00000400:00000010:0.0F:1773941317.351622:0:2340:0:(lib-cpt.c:111:cfs_cpt_table_alloc()) alloc '(cptab)': 56 at 0000000038f689a7. 00000400:00000010:0.0:1773941317.351631:0:2340:0:(lib-cpt.c:120:cfs_cpt_table_alloc()) alloc '(cptab->ctb_nodemask)': 128 at 0000000070c9c14c. 00000400:00000010:0.0:1773941317.351633:0:2340:0:(lib-cpt.c:124:cfs_cpt_table_alloc()) alloc '(cptab->ctb_cpu2cpt)': 16 at 000000002b1a6619. 00000400:00000010:0.0:1773941317.351635:0:2340:0:(lib-cpt.c:131:cfs_cpt_table_alloc()) alloc '(cptab->ctb_node2cpt)': 4 at 00000000aad232ce. 00000400:00000010:0.0:1773941317.351636:0:2340:0:(lib-cpt.c:138:cfs_cpt_table_alloc()) alloc '(cptab->ctb_parts)': 64 at 0000000091af2675. 00000400:00000010:0.0:1773941317.351638:0:2340:0:(lib-cpt.c:150:cfs_cpt_table_alloc()) alloc '(part->cpt_nodemask)': 128 at 0000000049840b94. 00000400:00000010:0.0:1773941317.351639:0:2340:0:(lib-cpt.c:154:cfs_cpt_table_alloc()) alloc '(part->cpt_distance)': 8 at 0000000050fb8777. 00000400:00000010:0.0:1773941317.351640:0:2340:0:(lib-cpt.c:150:cfs_cpt_table_alloc()) alloc '(part->cpt_nodemask)': 128 at 0000000054cac013. 00000400:00000010:0.0:1773941317.351640:0:2340:0:(lib-cpt.c:154:cfs_cpt_table_alloc()) alloc '(part->cpt_distance)': 8 at 0000000047105cb0. 00000400:00000010:3.0F:1773941318.107036:0:2340:0:(lock.c:44:cfs_percpt_lock_create()) alloc '(pcl)': 24 at 00000000a49d4672. 00000400:00000010:3.0:1773941318.107054:0:2340:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000000de71b38. 00000400:00000010:3.0:1773941318.107058:0:2340:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f6cba163. 00000400:00000010:3.0:1773941318.107060:0:2340:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002ed72466. 00000400:00000010:3.0:1773941318.107062:0:2340:0:(lock.c:44:cfs_percpt_lock_create()) alloc '(pcl)': 24 at 00000000963248b3. 00000400:00000010:3.0:1773941318.107064:0:2340:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000001c3a1257. 00000400:00000010:3.0:1773941318.107065:0:2340:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000006b6c8cca. 00000400:00000010:3.0:1773941318.107067:0:2340:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000006353288f. 00000400:00000200:3.0:1773941318.107192:0:2340:0:(api-ni.c:982:lnet_register_lnd()) lo LND registered 00000400:00000200:0.0:1773941318.160419:0:2356:0:(api-ni.c:982:lnet_register_lnd()) tcp LND registered 00000020:02000000:1.0F:1773941318.305832:0:2369:0:(class_obd.c:784:obdclass_init()) Lustre: Build Version: 2.17.50_193_gf5bca74 00000020:00000010:1.0:1773941318.308100:0:2369:0:(lustre_handles.c:148:class_handle_init()) kmalloced '(handle_hash)': 2097152 at 0000000013173e0b. 00000001:00000010:1.0:1773941318.308768:0:2369:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at 0000000059718daa. 00000001:00000010:1.0:1773941318.308774:0:2369:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at 0000000099f225c2. 00000001:00000010:1.0:1773941318.308777:0:2369:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000c3385ff2. 00000001:00000010:1.0:1773941318.308780:0:2369:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 000000008c72d15a. 00000001:00000010:1.0:1773941318.308782:0:2369:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 000000000929812b. 00000001:00000010:1.0:1773941318.308784:0:2369:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 0000000013d9429c. 00000020:00000010:1.0:1773941318.308822:0:2369:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000006148f6f2. 00000020:00000010:1.0:1773941318.308826:0:2369:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 0000000085faa7a5. 00000020:00000010:1.0:1773941318.308829:0:2369:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000fff5de5e. 00000020:00000010:1.0:1773941318.308831:0:2369:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000008cb3d212. 00000001:00000010:1.0:1773941318.308834:0:2369:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 0000000023a7250f. 00000020:00000010:1.0:1773941318.308872:0:2369:0:(cl_object.c:1176:cl_global_init()) kmalloced '(cl_envs)': 192 at 00000000964eea4d. 00000020:00000010:1.0:1773941318.309022:0:2369:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000ea83e89b. 00000020:00000010:1.0:1773941318.309026:0:2369:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000280a7129. 00000020:00000010:1.0:1773941318.309029:0:2369:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 0000000013279421. 00000020:00000010:1.0:1773941318.309031:0:2369:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000d22bf1a7. 00000020:00000010:1.0:1773941318.309034:0:2369:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000c9d2864e. 00000020:00000010:1.0:1773941318.309036:0:2369:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000005ac070d6. 00000020:00000010:1.0:1773941318.309038:0:2369:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 000000005d471ca4. 00000020:00000010:1.0:1773941318.309041:0:2369:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000b3fae084. 00000020:00000010:1.0:1773941318.309044:0:2369:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000e14b625b. 00000020:00000010:1.0:1773941318.309046:0:2369:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 0000000067cef5bb. 00000020:00000010:1.0:1773941318.309047:0:2369:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 0000000065fb555e. 00000020:00000010:1.0:1773941318.309049:0:2369:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000002ff07403. 00000020:00000010:1.0:1773941318.309052:0:2369:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000106c956b. 00000020:00000010:1.0:1773941318.309053:0:2369:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000002e86b155. 00000020:00000010:1.0:1773941318.309055:0:2369:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000a8414906. 00000020:00000010:1.0:1773941318.309057:0:2369:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000ad626b37. 02000000:00000010:1.0:1773941318.309062:0:2369:0:(page_pools.c:1061:obd_pool_init()) kmalloced '(page_pools)': 112 at 00000000a7f0a653. 02000000:00000010:1.0:1773941318.309065:0:2369:0:(page_pools.c:1065:obd_pool_init()) kmalloced '(pool_shrinkers)': 112 at 0000000009d3fd9e. 02000000:00000010:1.0:1773941318.309075:0:2369:0:(page_pools.c:1072:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 00000000684052cb. 02000000:00000010:1.0:1773941318.309081:0:2369:0:(page_pools.c:1032:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 1056 at 00000000a9769cd5. 00000001:00000010:1.0:1773941318.309084:0:2369:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 00000000d0b379ed. 00000020:00000010:1.0:1773941318.309124:0:2369:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 000000001c1bb587. 00000020:00000010:1.0:1773941318.309133:0:2369:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 000000001c1bb587. 02000000:00000010:1.0:1773941318.309137:0:2369:0:(page_pools.c:1072:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 000000007c3dcf4b. 02000000:00000010:1.0:1773941318.309140:0:2369:0:(page_pools.c:1032:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 528 at 00000000d79de0d0. 00000001:00000010:1.0:1773941318.309142:0:2369:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 00000000241c1770. 00000020:00000010:1.0:1773941318.309157:0:2369:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 000000001c1bb587. 00000020:00000010:1.0:1773941318.309163:0:2369:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 000000001c1bb587. 02000000:00000010:1.0:1773941318.309166:0:2369:0:(page_pools.c:1072:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 000000008060b088. 02000000:00000010:1.0:1773941318.309168:0:2369:0:(page_pools.c:1032:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 264 at 00000000f7c9ce23. 00000001:00000010:1.0:1773941318.309170:0:2369:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 000000001cdcf7c5. 00000020:00000010:1.0:1773941318.309184:0:2369:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 000000001c1bb587. 00000020:00000010:1.0:1773941318.309190:0:2369:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 000000001c1bb587. 02000000:00000010:1.0:1773941318.309193:0:2369:0:(page_pools.c:1072:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 000000004101e457. 02000000:00000010:1.0:1773941318.309194:0:2369:0:(page_pools.c:1032:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 136 at 000000001c1bb587. 00000001:00000010:1.0:1773941318.309195:0:2369:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 0000000059649366. 00000020:00000010:1.0:1773941318.309210:0:2369:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 00000000b2ec1b2a. 00000020:00000010:1.0:1773941318.309216:0:2369:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 00000000b2ec1b2a. 02000000:00000010:1.0:1773941318.309221:0:2369:0:(page_pools.c:1072:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 00000000235ba5af. 02000000:00000010:1.0:1773941318.309223:0:2369:0:(page_pools.c:1032:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 72 at 00000000c6ce88bc. 00000001:00000010:1.0:1773941318.309225:0:2369:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 000000001f33e772. 00000020:00000010:1.0:1773941318.309250:0:2369:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 00000000c5186b21. 00000020:00000010:1.0:1773941318.309256:0:2369:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 00000000c5186b21. 02000000:00000010:1.0:1773941318.309259:0:2369:0:(page_pools.c:1072:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 00000000876bbd27. 02000000:00000010:1.0:1773941318.309261:0:2369:0:(page_pools.c:1032:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 40 at 00000000cdac4f2d. 00000001:00000010:1.0:1773941318.309263:0:2369:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 000000008b529617. 00000020:00000010:1.0:1773941318.309295:0:2369:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 00000000312039d3. 00000020:00000010:1.0:1773941318.309301:0:2369:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 00000000312039d3. 02000000:00000010:1.0:1773941318.309304:0:2369:0:(page_pools.c:1072:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 00000000e4069c62. 02000000:00000010:1.0:1773941318.309307:0:2369:0:(page_pools.c:1032:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 24 at 0000000002fffefb. 00000001:00000010:1.0:1773941318.309313:0:2369:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 0000000021dd1c37. 00000020:00000010:1.0:1773941318.309329:0:2369:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 00000000312039d3. 00000020:00000010:1.0:1773941318.309334:0:2369:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 00000000312039d3. 02000000:00000010:1.0:1773941318.309337:0:2369:0:(page_pools.c:1072:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 000000003927208b. 02000000:00000010:1.0:1773941318.309339:0:2369:0:(page_pools.c:1032:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 16 at 000000002ca8fd26. 00000001:00000010:1.0:1773941318.309348:0:2369:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 000000004193bf87. 00000020:00000010:1.0:1773941318.309363:0:2369:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 00000000312039d3. 00000020:00000010:1.0:1773941318.309369:0:2369:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 00000000312039d3. 02000000:00000010:1.0:1773941318.309372:0:2369:0:(page_pools.c:1072:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 00000000e09ad431. 02000000:00000010:1.0:1773941318.309374:0:2369:0:(page_pools.c:1032:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 8 at 000000003ffc3c3a. 00000001:00000010:1.0:1773941318.309376:0:2369:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 00000000d104081b. 00000020:00000010:1.0:1773941318.309389:0:2369:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 00000000312039d3. 00000020:00000010:1.0:1773941318.309394:0:2369:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 00000000312039d3. 02000000:00000010:1.0:1773941318.309397:0:2369:0:(page_pools.c:1072:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 00000000997f0988. 02000000:00000010:1.0:1773941318.309399:0:2369:0:(page_pools.c:1032:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 8 at 00000000b3ec79fd. 00000001:00000010:1.0:1773941318.309401:0:2369:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 0000000029a13747. 00000020:00000010:1.0:1773941318.309423:0:2369:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 00000000c14ca34c. 00000020:00000010:1.0:1773941318.309436:0:2369:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 00000000c14ca34c. 02000000:00000010:1.0:1773941318.309440:0:2369:0:(page_pools.c:1072:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 000000006d6e9fe1. 02000000:00000010:1.0:1773941318.309442:0:2369:0:(page_pools.c:1032:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 8 at 0000000051f28dec. 00000001:00000010:1.0:1773941318.309444:0:2369:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 000000008a4bd338. 00000020:00000010:1.0:1773941318.309473:0:2369:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 00000000ce2175c1. 00000020:00000010:1.0:1773941318.309479:0:2369:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 00000000ce2175c1. 02000000:00000010:1.0:1773941318.309483:0:2369:0:(page_pools.c:1072:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 000000008a2d62a0. 02000000:00000010:1.0:1773941318.309485:0:2369:0:(page_pools.c:1032:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 8 at 0000000073a5d9bf. 00000001:00000010:1.0:1773941318.309487:0:2369:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 0000000055f6a92c. 00000020:00000010:1.0:1773941318.309500:0:2369:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 00000000ce2175c1. 00000020:00000010:1.0:1773941318.309505:0:2369:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 00000000ce2175c1. 02000000:00000010:1.0:1773941318.309508:0:2369:0:(page_pools.c:1072:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 00000000783eec60. 02000000:00000010:1.0:1773941318.309510:0:2369:0:(page_pools.c:1032:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 8 at 00000000ac093460. 00000001:00000010:1.0:1773941318.309512:0:2369:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 00000000425fca0f. 00000020:00000010:1.0:1773941318.309530:0:2369:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 00000000ce2175c1. 00000020:00000010:1.0:1773941318.309536:0:2369:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 00000000ce2175c1. 02000000:00000010:1.0:1773941318.309539:0:2369:0:(page_pools.c:1072:obd_pool_init()) kmalloced '(page_pools[pool_order])': 296 at 00000000f5c38678. 02000000:00000010:1.0:1773941318.309541:0:2369:0:(page_pools.c:1032:pool_ptrs_alloc()) kmalloced '(pool->opp_ptr_pages)': 8 at 0000000028410533. 00000001:00000010:1.0:1773941318.309543:0:2369:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 000000000b9ee48a. 00000020:00000010:1.0:1773941318.309565:0:2369:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 00000000d14b6b5a. 00000020:00000010:1.0:1773941318.309572:0:2369:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 00000000d14b6b5a. 00000400:00000010:0.0:1773941318.582756:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000d71e3237. 00000400:00000010:0.0:1773941318.582767:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f84737dd. 00000400:00000010:0.0:1773941318.582769:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000cadbbc33. 00000100:00000010:0.0:1773941318.582774:0:2384:0:(service.c:3571:ptlrpc_hr_init()) kmalloced '(hrp->hrp_thrs)': 96 at 0000000041221002. 00000100:00000010:0.0:1773941318.582776:0:2384:0:(service.c:3571:ptlrpc_hr_init()) kmalloced '(hrp->hrp_thrs)': 96 at 00000000417d592a. 00000100:00000010:0.0:1773941318.583010:0:2386:0:(service.c:3214:ptlrpc_hr_main()) kmalloced '(env)': 56 at 00000000dca19fd0. 00000020:00000010:0.0:1773941318.583022:0:2386:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000a7415859. 00000020:00000010:0.0:1773941318.583027:0:2386:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 0000000098d9de0b. 00000040:00000010:0.0:1773941318.583032:0:2386:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 0000000035da4ccc. 00000020:00000010:0.0:1773941318.583037:0:2386:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 000000002e7b1d42. 00000020:00000010:0.0:1773941318.583041:0:2386:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 00000000cf7be600. 00000100:00000010:0.0:1773941318.583268:0:2387:0:(service.c:3214:ptlrpc_hr_main()) kmalloced '(env)': 56 at 000000002223a175. 00000020:00000010:2.0F:1773941318.583304:0:2387:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000f211f8db. 00000020:00000010:2.0:1773941318.583316:0:2387:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000c20037ac. 00000040:00000010:2.0:1773941318.583319:0:2387:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 000000001d3cbcda. 00000020:00000010:2.0:1773941318.583324:0:2387:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000f2c04a48. 00000020:00000010:2.0:1773941318.583327:0:2387:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 0000000070f4d9be. 00000100:00000200:0.0:1773941318.583383:0:2384:0:(events.c:597:ptlrpc_ni_init()) My pid is: 3039 00000400:00000010:0.0:1773941318.583664:0:2384:0:(api-ni.c:679:lnet_create_remote_nets_table()) alloc '(hash)': 1024 at 0000000002903da0. 00000400:00000010:0.0:1773941318.583669:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000c3a9dea2. 00000400:00000010:0.0:1773941318.583672:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 000000009d82c0cf. 00000400:00000010:0.0:1773941318.583674:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 00000000c0dc02a7. 00000400:00000010:0.0:1773941318.583677:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000c9c9b96b. 00000400:00000010:0.0:1773941318.583680:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 00000000424f0211. 00000400:00000010:0.0:1773941318.583682:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 00000000b9c16863. 00000400:00000010:0.0:1773941318.583688:0:2384:0:(peer.c:108:lnet_peer_tables_create()) alloc '(hash)': 8192 at 000000003ad39747. 00000400:00000010:0.0:1773941318.583695:0:2384:0:(peer.c:108:lnet_peer_tables_create()) alloc '(hash)': 8192 at 000000000f943b71. 00000400:00000010:0.0:1773941318.583700:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000016eff723. 00000400:00000010:0.0:1773941318.583702:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 000000008f590032. 00000400:00000010:0.0:1773941318.583709:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 00000000fd0184d9. 00000400:00000010:0.0:1773941318.583712:0:2384:0:(lib-msg.c:1260:lnet_msg_container_setup()) alloc '(container->msc_finalizers)': 16 at 00000000fcbe5f29. 00000400:00000010:0.0:1773941318.583715:0:2384:0:(lib-msg.c:1270:lnet_msg_container_setup()) alloc '(container->msc_resenders)': 16 at 0000000018700646. 00000400:00000010:0.0:1773941318.583718:0:2384:0:(lib-msg.c:1260:lnet_msg_container_setup()) alloc '(container->msc_finalizers)': 16 at 00000000de16dab9. 00000400:00000010:0.0:1773941318.583719:0:2384:0:(lib-msg.c:1270:lnet_msg_container_setup()) alloc '(container->msc_resenders)': 16 at 000000005e50d88a. 00000400:00000010:0.0:1773941318.583788:0:2384:0:(api-ni.c:1174:lnet_res_container_setup()) alloc '(rec->rec_lh_hash)': 65536 at 000000001f93b8f1. 00000400:00000010:0.0:1773941318.583805:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000700f59fa. 00000400:00000010:0.0:1773941318.583807:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002e1b9076. 00000400:00000010:0.0:1773941318.583809:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ed829d99. 00000400:00000010:0.0:1773941318.583867:0:2384:0:(api-ni.c:1174:lnet_res_container_setup()) alloc '(rec->rec_lh_hash)': 65536 at 00000000cd5d5b3b. 00000400:00000010:0.0:1773941318.583944:0:2384:0:(api-ni.c:1174:lnet_res_container_setup()) alloc '(rec->rec_lh_hash)': 65536 at 00000000f6875fcf. 00000400:00000010:0.0:1773941318.583995:0:2384:0:(lib-ptl.c:867:lnet_portals_create()) alloc '(the_lnet.ln_portals)': 512 at 0000000047020340. 00000400:00000010:0.0:1773941318.583999:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000008d904900. 00000400:00000010:0.0:1773941318.584002:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000f2ec89ed. 00000400:00000010:0.0:1773941318.584005:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000000b1660f4. 00000400:00000010:0.0:1773941318.584007:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000061c91af8. 00000400:00000010:0.0:1773941318.584019:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000007b2913bc. 00000400:00000010:0.0:1773941318.584025:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000066a61002. 00000400:00000010:0.0:1773941318.584028:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000e7406728. 00000400:00000010:0.0:1773941318.584031:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000007aeca09d. 00000400:00000010:0.0:1773941318.584033:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f77ceef0. 00000400:00000010:0.0:1773941318.584035:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000e5fbdf61. 00000400:00000010:0.0:1773941318.584038:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000012328680. 00000400:00000010:0.0:1773941318.584041:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000007e1b7ab2. 00000400:00000010:0.0:1773941318.584043:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000003674d35e. 00000400:00000010:0.0:1773941318.584044:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000006e9c61ac. 00000400:00000010:0.0:1773941318.584045:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001e1d0151. 00000400:00000010:0.0:1773941318.584046:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000083d78ae2. 00000400:00000010:0.0:1773941318.584052:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000010c77fc5. 00000400:00000010:0.0:1773941318.584055:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000217501a5. 00000400:00000010:0.0:1773941318.584056:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000c82e7f40. 00000400:00000010:0.0:1773941318.584060:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000c62d2318. 00000400:00000010:0.0:1773941318.584061:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000067311017. 00000400:00000010:0.0:1773941318.584062:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002e7fd003. 00000400:00000010:0.0:1773941318.584066:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000bc647821. 00000400:00000010:0.0:1773941318.584068:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e717f7da. 00000400:00000010:0.0:1773941318.584070:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000e5d8d08c. 00000400:00000010:0.0:1773941318.584071:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000eff275b9. 00000400:00000010:0.0:1773941318.584072:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000917754a5. 00000400:00000010:0.0:1773941318.584072:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000041666657. 00000400:00000010:0.0:1773941318.584079:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e5740e3c. 00000400:00000010:0.0:1773941318.584084:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000004146879f. 00000400:00000010:0.0:1773941318.584085:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000003b247e39. 00000400:00000010:0.0:1773941318.584086:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000775b8117. 00000400:00000010:0.0:1773941318.584087:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000008181f27e. 00000400:00000010:0.0:1773941318.584088:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000040250f96. 00000400:00000010:0.0:1773941318.584091:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f5a296a0. 00000400:00000010:0.0:1773941318.584095:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000008d858e81. 00000400:00000010:0.0:1773941318.584096:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000ca14cf0d. 00000400:00000010:0.0:1773941318.584097:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000003ab16829. 00000400:00000010:0.0:1773941318.584098:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ada4c4e3. 00000400:00000010:0.0:1773941318.584099:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000004823c733. 00000400:00000010:0.0:1773941318.584105:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000001f84a407. 00000400:00000010:0.0:1773941318.584108:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e808aa61. 00000400:00000010:0.0:1773941318.584110:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000a2abac59. 00000400:00000010:0.0:1773941318.584110:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000001c1e285e. 00000400:00000010:0.0:1773941318.584111:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000000bccb39f. 00000400:00000010:0.0:1773941318.584112:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000005fb798e0. 00000400:00000010:0.0:1773941318.584115:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000761eb72d. 00000400:00000010:0.0:1773941318.584117:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000328f0415. 00000400:00000010:0.0:1773941318.584119:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000001240fe11. 00000400:00000010:0.0:1773941318.584120:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000afbb1696. 00000400:00000010:0.0:1773941318.584120:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f0b16083. 00000400:00000010:0.0:1773941318.584121:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000002e257404. 00000400:00000010:0.0:1773941318.584130:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000007e442d2a. 00000400:00000010:0.0:1773941318.584134:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000c5f9c130. 00000400:00000010:0.0:1773941318.584135:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000006d5a3579. 00000400:00000010:0.0:1773941318.584136:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000009c78935c. 00000400:00000010:0.0:1773941318.584137:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000004cd2512c. 00000400:00000010:0.0:1773941318.584138:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000054c3e5b6. 00000400:00000010:0.0:1773941318.584142:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000406fda78. 00000400:00000010:0.0:1773941318.584145:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000040134fb1. 00000400:00000010:0.0:1773941318.584147:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000004924d7b8. 00000400:00000010:0.0:1773941318.584148:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000007a22689. 00000400:00000010:0.0:1773941318.584149:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000efc610f4. 00000400:00000010:0.0:1773941318.584150:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000728879b6. 00000400:00000010:0.0:1773941318.584155:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e55e5887. 00000400:00000010:0.0:1773941318.584159:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f2395b2d. 00000400:00000010:0.0:1773941318.584160:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 0000000022c6bc6c. 00000400:00000010:0.0:1773941318.584161:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000cdb7a49b. 00000400:00000010:0.0:1773941318.584162:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000edaf37f4. 00000400:00000010:0.0:1773941318.584164:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003bba97de. 00000400:00000010:0.0:1773941318.584166:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000009be69153. 00000400:00000010:0.0:1773941318.584169:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000097634084. 00000400:00000010:0.0:1773941318.584170:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000b1b8d310. 00000400:00000010:0.0:1773941318.584171:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000004c62a8ff. 00000400:00000010:0.0:1773941318.584172:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000090e98f68. 00000400:00000010:0.0:1773941318.584173:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000031c7a866. 00000400:00000010:0.0:1773941318.584178:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000005aec92e6. 00000400:00000010:0.0:1773941318.584182:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000007f458041. 00000400:00000010:0.0:1773941318.584183:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 0000000058e182b4. 00000400:00000010:0.0:1773941318.584184:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000edf9c25e. 00000400:00000010:0.0:1773941318.584185:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000721a535d. 00000400:00000010:0.0:1773941318.584187:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ca0a9f49. 00000400:00000010:0.0:1773941318.584190:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000009aefbdfe. 00000400:00000010:0.0:1773941318.584193:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000004fca7324. 00000400:00000010:0.0:1773941318.584194:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000870aee57. 00000400:00000010:0.0:1773941318.584199:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000009dbf66. 00000400:00000010:0.0:1773941318.584200:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b64aef29. 00000400:00000010:0.0:1773941318.584202:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ece3cf0d. 00000400:00000010:0.0:1773941318.584209:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e6cbf0d4. 00000400:00000010:0.0:1773941318.584213:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ca0ed3a7. 00000400:00000010:0.0:1773941318.584215:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000a5ddf610. 00000400:00000010:0.0:1773941318.584216:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000003d97ad1e. 00000400:00000010:0.0:1773941318.584217:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000000d7b6f12. 00000400:00000010:0.0:1773941318.584218:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007fb831d4. 00000400:00000010:0.0:1773941318.584221:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ea61a8ac. 00000400:00000010:0.0:1773941318.584225:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000008343e077. 00000400:00000010:0.0:1773941318.584226:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 0000000034b84c04. 00000400:00000010:0.0:1773941318.584227:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000ecc9233a. 00000400:00000010:0.0:1773941318.584228:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000078a8bc53. 00000400:00000010:0.0:1773941318.584229:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a6802b12. 00000400:00000010:0.0:1773941318.584236:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000a7703d4c. 00000400:00000010:0.0:1773941318.584240:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000007f987c70. 00000400:00000010:0.0:1773941318.584242:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000000d235898. 00000400:00000010:0.0:1773941318.584243:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000c9dd27f4. 00000400:00000010:0.0:1773941318.584243:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000184c8647. 00000400:00000010:0.0:1773941318.584244:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001e111cd1. 00000400:00000010:0.0:1773941318.584248:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000061ac8cb2. 00000400:00000010:0.0:1773941318.584252:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000a936f40b. 00000400:00000010:0.0:1773941318.584253:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000f3673089. 00000400:00000010:0.0:1773941318.584254:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000002f8923a1. 00000400:00000010:0.0:1773941318.584255:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f7b47af4. 00000400:00000010:0.0:1773941318.584255:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001008eb35. 00000400:00000010:0.0:1773941318.584262:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000d6a66bad. 00000400:00000010:0.0:1773941318.584266:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000088e98e65. 00000400:00000010:0.0:1773941318.584268:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000007cc6df34. 00000400:00000010:0.0:1773941318.584269:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000004916d00. 00000400:00000010:0.0:1773941318.584270:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ba3a1fd3. 00000400:00000010:0.0:1773941318.584271:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b1134e5f. 00000400:00000010:0.0:1773941318.584277:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000001454bbd7. 00000400:00000010:0.0:1773941318.584282:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000a745d9bb. 00000400:00000010:0.0:1773941318.584285:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000003fec4566. 00000400:00000010:0.0:1773941318.584287:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000ddc7ac4c. 00000400:00000010:0.0:1773941318.584289:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000dd31b163. 00000400:00000010:0.0:1773941318.584290:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000eac60941. 00000400:00000010:0.0:1773941318.584300:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000078259483. 00000400:00000010:0.0:1773941318.584306:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000073ab2d0a. 00000400:00000010:0.0:1773941318.584317:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000572911c6. 00000400:00000010:0.0:1773941318.584320:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000de9a967f. 00000400:00000010:0.0:1773941318.584323:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000008b617d92. 00000400:00000010:0.0:1773941318.584325:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000008719305e. 00000400:00000010:0.0:1773941318.584330:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b6fae989. 00000400:00000010:0.0:1773941318.584336:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000bb225af8. 00000400:00000010:0.0:1773941318.584338:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000791f28a1. 00000400:00000010:0.0:1773941318.584340:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000ee1d6b36. 00000400:00000010:0.0:1773941318.584342:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007c25cdc5. 00000400:00000010:0.0:1773941318.584343:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000c892dd2b. 00000400:00000010:0.0:1773941318.584353:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000018c85b98. 00000400:00000010:0.0:1773941318.584359:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000d4f500ac. 00000400:00000010:0.0:1773941318.584362:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000f0b4457c. 00000400:00000010:0.0:1773941318.584364:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000dd3d72f7. 00000400:00000010:0.0:1773941318.584365:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b1136c47. 00000400:00000010:0.0:1773941318.584367:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003543e828. 00000400:00000010:0.0:1773941318.584371:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000bf57a17f. 00000400:00000010:0.0:1773941318.584376:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000006c0ea407. 00000400:00000010:0.0:1773941318.584378:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000009c76b862. 00000400:00000010:0.0:1773941318.584379:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000b5d1a3be. 00000400:00000010:0.0:1773941318.584381:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d150707a. 00000400:00000010:0.0:1773941318.584382:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000861d0d2c. 00000400:00000010:0.0:1773941318.584391:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000002837ea96. 00000400:00000010:0.0:1773941318.584397:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000014a9e5a0. 00000400:00000010:0.0:1773941318.584399:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000009a769e22. 00000400:00000010:0.0:1773941318.584405:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000050bcb43a. 00000400:00000010:0.0:1773941318.584407:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000004891a90d. 00000400:00000010:0.0:1773941318.584409:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000021caa18d. 00000400:00000010:0.0:1773941318.584414:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000003f16fa6d. 00000400:00000010:0.0:1773941318.584419:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000771edfe9. 00000400:00000010:0.0:1773941318.584421:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 0000000025deee85. 00000400:00000010:0.0:1773941318.584423:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000c38f55ae. 00000400:00000010:0.0:1773941318.584425:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007ed1ee1b. 00000400:00000010:0.0:1773941318.584427:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000000aad65e. 00000400:00000010:0.0:1773941318.584437:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000731ecbd9. 00000400:00000010:0.0:1773941318.584443:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000062a62c54. 00000400:00000010:0.0:1773941318.584446:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 0000000050b4333c. 00000400:00000010:0.0:1773941318.584448:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000d3788e2c. 00000400:00000010:0.0:1773941318.584450:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000cff95c7e. 00000400:00000010:0.0:1773941318.584452:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000e1252a68. 00000400:00000010:0.0:1773941318.584456:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e9c34803. 00000400:00000010:0.0:1773941318.584460:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000aae5246d. 00000400:00000010:0.0:1773941318.584462:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 0000000033db3e8b. 00000400:00000010:0.0:1773941318.584464:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000051d20814. 00000400:00000010:0.0:1773941318.584466:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000386a0a99. 00000400:00000010:0.0:1773941318.584468:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000fcdace04. 00000400:00000010:0.0:1773941318.584478:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000046b5642c. 00000400:00000010:0.0:1773941318.584484:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000005ed57748. 00000400:00000010:0.0:1773941318.584487:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000be614586. 00000400:00000010:0.0:1773941318.584489:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000081abd5a4. 00000400:00000010:0.0:1773941318.584505:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000495e1af1. 00000400:00000010:0.0:1773941318.584508:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000005460e585. 00000400:00000010:0.0:1773941318.584513:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000283d38e6. 00000400:00000010:0.0:1773941318.584518:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000eeaf600d. 00000400:00000010:0.0:1773941318.584521:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 0000000083fe8e81. 00000400:00000010:0.0:1773941318.584523:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000054ea5b81. 00000400:00000010:0.0:1773941318.584525:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b93d9234. 00000400:00000010:0.0:1773941318.584526:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000dc5d842e. 00000400:00000010:0.0:1773941318.584538:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000aa97437a. 00000400:00000010:0.0:1773941318.584542:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000036f37027. 00000400:00000010:0.0:1773941318.584545:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000005c295514. 00000400:00000010:0.0:1773941318.584546:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000007a1e2474. 00000400:00000010:0.0:1773941318.584548:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007825034c. 00000400:00000010:0.0:1773941318.584549:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000005cbba02a. 00000400:00000010:0.0:1773941318.584553:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000007e460200. 00000400:00000010:0.0:1773941318.584556:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000a33c58eb. 00000400:00000010:0.0:1773941318.584558:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000c38d4cbd. 00000400:00000010:0.0:1773941318.584560:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000a2dea437. 00000400:00000010:0.0:1773941318.584561:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b50de586. 00000400:00000010:0.0:1773941318.584563:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000006d7b1ce6. 00000400:00000010:0.0:1773941318.584572:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f46d5df9. 00000400:00000010:0.0:1773941318.584577:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000005ae259d6. 00000400:00000010:0.0:1773941318.584579:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000c5daa170. 00000400:00000010:0.0:1773941318.584581:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000006d221a6f. 00000400:00000010:0.0:1773941318.584582:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000e1deaa69. 00000400:00000010:0.0:1773941318.584583:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001fff0e2d. 00000400:00000010:0.0:1773941318.584587:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b00c3fbb. 00000400:00000010:0.0:1773941318.584591:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b63dde67. 00000400:00000010:0.0:1773941318.584592:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000001dd5f55c. 00000400:00000010:0.0:1773941318.584593:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000004db54d30. 00000400:00000010:0.0:1773941318.584594:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ca6e9766. 00000400:00000010:0.0:1773941318.584594:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000044bba019. 00000400:00000010:0.0:1773941318.584601:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000655db9bc. 00000400:00000010:0.0:1773941318.584605:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000edb355f3. 00000400:00000010:0.0:1773941318.584607:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000006244a0eb. 00000400:00000010:0.0:1773941318.584607:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000daa4097a. 00000400:00000010:0.0:1773941318.584608:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000035a714a8. 00000400:00000010:0.0:1773941318.584609:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f7ddcacc. 00000400:00000010:0.0:1773941318.584613:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e1f50041. 00000400:00000010:0.0:1773941318.584616:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000002525abc4. 00000400:00000010:0.0:1773941318.584618:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000edd568d9. 00000400:00000010:0.0:1773941318.584623:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000776c7b52. 00000400:00000010:0.0:1773941318.584625:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000005108c355. 00000400:00000010:0.0:1773941318.584637:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003431376d. 00000400:00000010:0.0:1773941318.584648:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000c29ff0f9. 00000400:00000010:0.0:1773941318.584654:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000006215bcea. 00000400:00000010:0.0:1773941318.584657:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000e3a3daa3. 00000400:00000010:0.0:1773941318.584659:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000005ad1cc7f. 00000400:00000010:0.0:1773941318.584661:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000009000576c. 00000400:00000010:0.0:1773941318.584662:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a8cdd2fa. 00000400:00000010:0.0:1773941318.584667:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000ddaac74d. 00000400:00000010:0.0:1773941318.584672:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000003b2ee0f5. 00000400:00000010:0.0:1773941318.584675:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 0000000036fc4e87. 00000400:00000010:0.0:1773941318.584677:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000081e27c2b. 00000400:00000010:0.0:1773941318.584679:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000043a4f78d. 00000400:00000010:0.0:1773941318.584681:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a6a6f29b. 00000400:00000010:0.0:1773941318.584690:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000008d4a20f0. 00000400:00000010:0.0:1773941318.584696:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e133e1da. 00000400:00000010:0.0:1773941318.584698:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000004012f515. 00000400:00000010:0.0:1773941318.584700:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000517b6f4f. 00000400:00000010:0.0:1773941318.584702:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000006da08715. 00000400:00000010:0.0:1773941318.584704:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b52b24fb. 00000400:00000010:0.0:1773941318.584708:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000002e19b279. 00000400:00000010:0.0:1773941318.584712:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000002847bc8c. 00000400:00000010:0.0:1773941318.584714:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 0000000065ca039f. 00000400:00000010:0.0:1773941318.584716:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000233ae5d4. 00000400:00000010:0.0:1773941318.584717:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000008e201ba1. 00000400:00000010:0.0:1773941318.584719:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000006d7c8023. 00000400:00000010:0.0:1773941318.584728:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000e6ba6a3a. 00000400:00000010:0.0:1773941318.584733:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000a62119b8. 00000400:00000010:0.0:1773941318.584735:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000003ba4c383. 00000400:00000010:0.0:1773941318.584737:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000d86efda0. 00000400:00000010:0.0:1773941318.584738:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f660f3a0. 00000400:00000010:0.0:1773941318.584740:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000374f7fc2. 00000400:00000010:0.0:1773941318.584749:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b25d69ae. 00000400:00000010:0.0:1773941318.584754:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000309831ad. 00000400:00000010:0.0:1773941318.584756:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000007f26eb1f. 00000400:00000010:0.0:1773941318.584758:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000009e78ba49. 00000400:00000010:0.0:1773941318.584760:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007b8931c3. 00000400:00000010:0.0:1773941318.584761:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d8fd50ed. 00000400:00000010:0.0:1773941318.584770:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f01bf369. 00000400:00000010:0.0:1773941318.584775:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000002fea5c51. 00000400:00000010:0.0:1773941318.584777:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000055ec422. 00000400:00000010:0.0:1773941318.584779:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000009c87f71f. 00000400:00000010:0.0:1773941318.584781:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000006b0559f7. 00000400:00000010:0.0:1773941318.584782:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000006734a62a. 00000400:00000010:0.0:1773941318.584786:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000a8b4fa39. 00000400:00000010:0.0:1773941318.584791:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b7a604c7. 00000400:00000010:0.0:1773941318.584793:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000281d1ecb. 00000400:00000010:0.0:1773941318.584794:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000007219d2bb. 00000400:00000010:0.0:1773941318.584795:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000008b13abfe. 00000400:00000010:0.0:1773941318.584795:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000da74a800. 00000400:00000010:0.0:1773941318.584802:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000de5dfd09. 00000400:00000010:0.0:1773941318.584806:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000b70c8983. 00000400:00000010:0.0:1773941318.584808:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000a0caefdf. 00000400:00000010:0.0:1773941318.584810:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000225824dd. 00000400:00000010:0.0:1773941318.584811:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000004677d86d. 00000400:00000010:0.0:1773941318.584813:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ab896217. 00000400:00000010:0.0:1773941318.584817:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000009730a239. 00000400:00000010:0.0:1773941318.584821:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000796feab6. 00000400:00000010:0.0:1773941318.584823:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000005143d40d. 00000400:00000010:0.0:1773941318.584825:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000006e27e5db. 00000400:00000010:0.0:1773941318.584827:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000dba5ecdd. 00000400:00000010:0.0:1773941318.584829:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000872dbf4a. 00000400:00000010:0.0:1773941318.584839:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000031b3280a. 00000400:00000010:0.0:1773941318.584846:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000017c76b68. 00000400:00000010:0.0:1773941318.584849:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000002064bd4f. 00000400:00000010:0.0:1773941318.584855:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000005d8b4c3c. 00000400:00000010:0.0:1773941318.584857:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000194b1db7. 00000400:00000010:0.0:1773941318.584859:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000cf5db984. 00000400:00000010:0.0:1773941318.584863:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000aeefb4f6. 00000400:00000010:0.0:1773941318.584868:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000697277a9. 00000400:00000010:0.0:1773941318.584870:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000cf614b9c. 00000400:00000010:0.0:1773941318.584872:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000d1373939. 00000400:00000010:0.0:1773941318.584873:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000c8649056. 00000400:00000010:0.0:1773941318.584875:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000099d4843. 00000400:00000010:0.0:1773941318.584884:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000007418de1a. 00000400:00000010:0.0:1773941318.584888:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000098d51526. 00000400:00000010:0.0:1773941318.584890:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000f0b0a7e3. 00000400:00000010:0.0:1773941318.584900:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000054616ac7. 00000400:00000010:0.0:1773941318.584902:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b3e2fbed. 00000400:00000010:0.0:1773941318.584904:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000e8fa7bcf. 00000400:00000010:0.0:1773941318.584907:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000009bc902b9. 00000400:00000010:0.0:1773941318.584911:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000bf2ea063. 00000400:00000010:0.0:1773941318.584913:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000b887a128. 00000400:00000010:0.0:1773941318.584915:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000d22995e9. 00000400:00000010:0.0:1773941318.584917:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000f164523b. 00000400:00000010:0.0:1773941318.584918:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003b146442. 00000400:00000010:0.0:1773941318.584926:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000004a9928dd. 00000400:00000010:0.0:1773941318.584930:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000d327c71e. 00000400:00000010:0.0:1773941318.584933:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000007d14291f. 00000400:00000010:0.0:1773941318.584934:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000040d85784. 00000400:00000010:0.0:1773941318.584936:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001fc54d2f. 00000400:00000010:0.0:1773941318.584937:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003ec8b88b. 00000400:00000010:0.0:1773941318.584940:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000008b195caf. 00000400:00000010:0.0:1773941318.584944:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000828d7ac2. 00000400:00000010:0.0:1773941318.584946:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000b67ff080. 00000400:00000010:0.0:1773941318.584948:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000b5f7797c. 00000400:00000010:0.0:1773941318.584950:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a60ebbce. 00000400:00000010:0.0:1773941318.584952:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b80ae150. 00000400:00000010:0.0:1773941318.584982:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000a1df1c6d. 00000400:00000010:0.0:1773941318.584988:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000070a3bc91. 00000400:00000010:0.0:1773941318.584991:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000a0abe39c. 00000400:00000010:0.0:1773941318.584993:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000030e24575. 00000400:00000010:0.0:1773941318.584996:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000018136572. 00000400:00000010:0.0:1773941318.584999:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000001fe50f6a. 00000400:00000010:0.0:1773941318.585002:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000003b7b4dfe. 00000400:00000010:0.0:1773941318.585006:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000003907170. 00000400:00000010:0.0:1773941318.585009:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000a6dd990c. 00000400:00000010:0.0:1773941318.585012:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000002d3a700e. 00000400:00000010:0.0:1773941318.585013:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000007bb36a42. 00000400:00000010:0.0:1773941318.585015:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000328da180. 00000400:00000010:0.0:1773941318.585024:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000dfa7fdd2. 00000400:00000010:0.0:1773941318.585028:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000151f17f6. 00000400:00000010:0.0:1773941318.585030:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000f1612edb. 00000400:00000010:0.0:1773941318.585032:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000b50c383d. 00000400:00000010:0.0:1773941318.585034:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000b5933229. 00000400:00000010:0.0:1773941318.585035:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d1283802. 00000400:00000010:0.0:1773941318.585038:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000a61141ad. 00000400:00000010:0.0:1773941318.585042:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000009e2413e9. 00000400:00000010:0.0:1773941318.585044:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 0000000014ba0449. 00000400:00000010:0.0:1773941318.585046:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000002532c3bc. 00000400:00000010:0.0:1773941318.585047:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000088a250fb. 00000400:00000010:0.0:1773941318.585049:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000004692ed25. 00000400:00000010:0.0:1773941318.585056:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000050fc5c43. 00000400:00000010:0.0:1773941318.585060:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000cb3313c7. 00000400:00000010:0.0:1773941318.585063:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000c08b73a8. 00000400:00000010:0.0:1773941318.585064:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000008e0df6a2. 00000400:00000010:0.0:1773941318.585066:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000006ceab080. 00000400:00000010:0.0:1773941318.585067:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000003c17e8eb. 00000400:00000010:0.0:1773941318.585071:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000003efe4c7c. 00000400:00000010:0.0:1773941318.585074:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000f0b3939a. 00000400:00000010:0.0:1773941318.585075:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 0000000007d7c0e8. 00000400:00000010:0.0:1773941318.585079:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000006c3dc139. 00000400:00000010:0.0:1773941318.585080:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a4f49fce. 00000400:00000010:0.0:1773941318.585081:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000008a041c02. 00000400:00000010:0.0:1773941318.585089:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000037fb15d6. 00000400:00000010:0.0:1773941318.585093:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000001da0ce39. 00000400:00000010:0.0:1773941318.585095:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000003db23fc0. 00000400:00000010:0.0:1773941318.585096:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000930a6116. 00000400:00000010:0.0:1773941318.585097:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000ac7a9c47. 00000400:00000010:0.0:1773941318.585098:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000933d3741. 00000400:00000010:0.0:1773941318.585101:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000007e19fba3. 00000400:00000010:0.0:1773941318.585105:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000008660df32. 00000400:00000010:0.0:1773941318.585106:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000006d747d8b. 00000400:00000010:0.0:1773941318.585107:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000004fd061ef. 00000400:00000010:0.0:1773941318.585108:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000461371cc. 00000400:00000010:0.0:1773941318.585109:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d283c93f. 00000400:00000010:0.0:1773941318.585116:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000003a204932. 00000400:00000010:0.0:1773941318.585120:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000091b6aed1. 00000400:00000010:0.0:1773941318.585121:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000f373cb48. 00000400:00000010:0.0:1773941318.585122:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000dc5a094b. 00000400:00000010:0.0:1773941318.585129:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000004e2f8422. 00000400:00000010:0.0:1773941318.585130:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000d4e85ed6. 00000400:00000010:0.0:1773941318.585134:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000071998086. 00000400:00000010:0.0:1773941318.585137:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000002caf3175. 00000400:00000010:0.0:1773941318.585138:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 00000000e2c7e91c. 00000400:00000010:0.0:1773941318.585139:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000008ad9ceed. 00000400:00000010:0.0:1773941318.585140:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000784b2283. 00000400:00000010:0.0:1773941318.585141:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000066b6fc2c. 00000400:00000010:0.0:1773941318.585147:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000040ed6af8. 00000400:00000010:0.0:1773941318.585151:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 00000000c5ddae7b. 00000400:00000010:0.0:1773941318.585160:0:2384:0:(lib-ptl.c:874:lnet_portals_create()) alloc '(the_lnet.ln_portals[i])': 88 at 000000009331b2d4. 00000400:00000010:0.0:1773941318.585162:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000755647fc. 00000400:00000010:0.0:1773941318.585163:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000071cdceec. 00000400:00000010:0.0:1773941318.585164:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000006f8ae4c. 00000400:00000010:0.0:1773941318.585171:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 0000000083326b2d. 00000400:00000010:0.0:1773941318.585174:0:2384:0:(lib-ptl.c:818:lnet_ptl_setup()) alloc '(mhash)': 4112 at 000000006916acfb. 00000400:00000010:0.0:1773941318.585176:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000037e956a0. 00000400:00000010:0.0:1773941318.585178:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000038c80fb8. 00000400:00000010:0.0:1773941318.585179:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000066a5092a. 00000400:00000010:0.0:1773941318.585182:0:2384:0:(config.c:344:lnet_net_alloc()) alloc '(net)': 176 at 00000000a016b4a6. 00000400:00000010:0.0:1773941318.585184:0:2384:0:(config.c:425:lnet_ni_alloc_common()) alloc '(ni)': 448 at 0000000086f78e68. 00000400:00000010:0.0:1773941318.585185:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000f0c3febf. 00000400:00000010:0.0:1773941318.585186:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000005afc807a. 00000400:00000010:0.0:1773941318.585187:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 0000000032c49767. 00000400:00000010:0.0:1773941318.585188:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000c8eace3b. 00000400:00000010:0.0:1773941318.585189:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000004b4cc875. 00000400:00000010:0.0:1773941318.585190:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000730942bf. 00000400:00000010:0.0:1773941318.585194:0:2384:0:(config.c:592:lnet_parse_networks()) alloc '(tokens)': 4 at 0000000009960a85. 00000400:00000010:0.0:1773941318.585198:0:2384:0:(config.c:344:lnet_net_alloc()) alloc '(net)': 176 at 00000000fc7f9209. 00000400:00000010:0.0:1773941318.585199:0:2384:0:(config.c:425:lnet_ni_alloc_common()) alloc '(ni)': 448 at 00000000ee09ce70. 00000400:00000010:0.0:1773941318.585201:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 0000000006a13713. 00000400:00000010:0.0:1773941318.585202:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000029d9f13. 00000400:00000010:0.0:1773941318.585203:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000582c4f0e. 00000400:00000010:0.0:1773941318.585204:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 000000001debaa7f. 00000400:00000010:0.0:1773941318.585205:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 000000008fc8f57f. 00000400:00000010:0.0:1773941318.585206:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000646e52bd. 00000400:00000010:0.0:1773941318.585207:0:2384:0:(config.c:804:lnet_parse_networks()) kfreed 'tokens': 4 at 0000000009960a85. 00000400:00000010:0.0:1773941318.585218:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000cb4a398d. 00000400:00000010:0.0:1773941318.585219:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 0000000093cd4fa7. 00000400:00000010:0.0:1773941318.585220:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 128 at 0000000009f2c3af. 00000800:00000200:2.0:1773941318.585866:0:2384:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_REGISTER, iface=lo ifindex 1 state 0 ns 00000000209354f3 00000800:00000200:2.0:1773941318.585870:0:2384:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UP, iface=lo ifindex 1 state 0 ns 00000000209354f3 00000800:00000200:2.0:1773941318.585872:0:2384:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_REGISTER, iface=ens2 ifindex 2 state 6 ns 00000000209354f3 00000800:00000200:2.0:1773941318.585873:0:2384:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UP, iface=ens2 ifindex 2 state 6 ns 00000000209354f3 00000800:00000200:2.0:1773941318.585875:0:2384:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_REGISTER, iface=lo ifindex 1 state 0 ns 000000002ed8b4a5 00000800:00000200:2.0:1773941318.585877:0:2384:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UP, iface=lo ifindex 1 state 0 ns 000000002ed8b4a5 00000800:00000010:2.0:1773941318.585883:0:2384:0:(socklnd.c:2622:ksocknal_startup()) alloc '(net)': 208 at 000000002b7dd4ec. 00000400:00000010:2.0:1773941318.585891:0:2384:0:(config.c:398:lnet_ni_add_interface()) alloc '(ni->ni_interface)': 5 at 0000000062afbeca. 00000800:00000200:2.0:1773941318.585896:0:2384:0:(socklnd_modparams.c:304:ksocklnd_lookup_conns_per_peer()) intf ens2 speed -1 00000800:00000010:0.0:1773941318.586052:0:2393:0:(socklnd_cb.c:1469:ksocknal_scheduler()) alloc '(rx_scratch_pgs)': 2048 at 0000000010f12342. 00000800:00000010:0.0:1773941318.586057:0:2393:0:(socklnd_cb.c:1476:ksocknal_scheduler()) alloc '(scratch_iov)': 4096 at 000000002cf30a6b. 00000800:00000010:0.0:1773941318.586157:0:2394:0:(socklnd_cb.c:1469:ksocknal_scheduler()) alloc '(rx_scratch_pgs)': 2048 at 0000000078b4ca74. 00000800:00000010:0.0:1773941318.586161:0:2394:0:(socklnd_cb.c:1476:ksocknal_scheduler()) alloc '(scratch_iov)': 4096 at 00000000fcd8a6e6. 00000800:00000010:0.0:1773941318.586270:0:2395:0:(socklnd_cb.c:1469:ksocknal_scheduler()) alloc '(rx_scratch_pgs)': 2048 at 00000000f305289c. 00000800:00000010:0.0:1773941318.586275:0:2395:0:(socklnd_cb.c:1476:ksocknal_scheduler()) alloc '(scratch_iov)': 4096 at 000000009fd05068. 00000400:00000200:2.0:1773941318.586376:0:2384:0:(config.c:1527:lnet_get_link_status_locked()) device idx 1 get_link 2 00000800:00000010:0.0:1773941318.586386:0:2396:0:(socklnd_cb.c:1469:ksocknal_scheduler()) alloc '(rx_scratch_pgs)': 2048 at 000000006bfe7976. 00000800:00000010:0.0:1773941318.586398:0:2396:0:(socklnd_cb.c:1476:ksocknal_scheduler()) alloc '(scratch_iov)': 4096 at 0000000012d7ffe8. 00000400:00000200:2.0:1773941318.586443:0:2384:0:(acceptor.c:209:lnet_acceptor_add_socket()) Added socket on ens2:988 (IP: 192.168.204.14) 00000400:02000000:2.0:1773941318.586453:0:2384:0:(api-ni.c:2704:lnet_startup_lndni()) Added LNI 192.168.204.14@tcp [8/256/0/180] 00000400:00000010:2.0:1773941318.588139:0:2384:0:(api-ni.c:1821:lnet_ping_buffer_alloc()) alloc '(pbuf)': 57 at 00000000164df97b. 00000400:00000010:2.0:1773941318.588146:0:2384:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000013398e6b. 00000400:00000010:2.0:1773941318.588164:0:2384:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000005bbe24e9. 00000400:00000200:2.0:1773941318.588175:0:2384:0:(lib-ptl.c:921:LNetSetLazyPortal()) Setting portal 0 lazy 00000400:00000010:2.0:1773941318.588179:0:2384:0:(api-ni.c:1821:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000005759e74f. 00000400:00000010:2.0:1773941318.588183:0:2384:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000025f467c2. 00000400:00000010:2.0:1773941318.588186:0:2384:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006d307180. 00000400:00000200:2.0:1773941318.588189:0:2384:0:(api-ni.c:2331:lnet_push_target_post()) posted push target 000000005759e74f 00000400:00000200:2.0:1773941318.588191:0:2384:0:(api-ni.c:2288:lnet_push_target_resize()) nbytes 272 success 00000400:00000010:2.0:1773941318.588194:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000f0c37aea. 00000400:00000010:2.0:1773941318.588197:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000e0e56f82. 00000400:00000010:2.0:1773941318.588199:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000c730d3b6. 00000400:00000010:2.0:1773941318.588201:0:2384:0:(lib-cpt.c:1345:cfs_percpt_alloc()) alloc '(arr)': 32 at 00000000d35347f5. 00000400:00000010:2.0:1773941318.588203:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000a459a140. 00000400:00000010:2.0:1773941318.588205:0:2384:0:(lib-cpt.c:1355:cfs_percpt_alloc()) alloc '(arr->va_ptrs[i])': 64 at 00000000633ebc92. 00000400:00000200:2.0:1773941318.588486:0:2384:0:(peer.c:4215:lnet_peer_discovery_start()) discovery start: 0 00000100:00000010:2.0:1773941318.588589:0:2384:0:(ptlrpcd.c:749:ptlrpcd_init()) kmalloced '(ptlrpcds)': 16 at 00000000a5123d35. 00000400:00000200:3.0:1773941318.588604:0:2399:0:(peer.c:4041:lnet_peer_discovery()) started 00000100:00000010:3.0:1773941318.588730:0:2400:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 0000000071a316af. 00000020:00000010:3.0:1773941318.588737:0:2400:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000000dd5e0e2. 00000020:00000010:3.0:1773941318.588741:0:2400:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 0000000021027219. 00000020:00000010:3.0:1773941318.588745:0:2400:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000b86133f1. 00000040:00000010:3.0:1773941318.588748:0:2400:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 0000000086ae159e. 00000020:00000010:3.0:1773941318.588753:0:2400:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000bc05c732. 00000020:00000010:3.0:1773941318.588756:0:2400:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000003e9fe135. 00000100:00000010:2.0:1773941318.588776:0:2384:0:(ptlrpcd.c:846:ptlrpcd_init()) kmalloced '(pd)': 392 at 000000003f1123f6. 00000100:00000010:2.0:1773941318.588780:0:2384:0:(ptlrpcd.c:538:ptlrpcd_partners()) kmalloced '(pc->pc_partners)': 8 at 000000008b965137. 00000100:00000010:2.0:1773941318.588782:0:2384:0:(ptlrpcd.c:538:ptlrpcd_partners()) kmalloced '(pc->pc_partners)': 8 at 00000000d1e2eee2. 00000100:00000010:0.0:1773941318.588912:0:2401:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 0000000068fc9de6. 00000020:00000010:0.0:1773941318.588917:0:2401:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000d37df6dc. 00000020:00000010:0.0:1773941318.588919:0:2401:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000d4deea01. 00000020:00000010:0.0:1773941318.588922:0:2401:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 0000000061d13ff6. 00000040:00000010:0.0:1773941318.588924:0:2401:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 000000005ffdab59. 00000020:00000010:0.0:1773941318.588928:0:2401:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 0000000074225d9c. 00000020:00000010:0.0:1773941318.588930:0:2401:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000be7f7700. 00000100:00000010:0.0:1773941318.589114:0:2402:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 000000000c80bf85. 00000020:00000010:0.0:1773941318.589119:0:2402:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000020d33fe6. 00000020:00000010:0.0:1773941318.589122:0:2402:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 0000000088cfa3e3. 00000020:00000010:0.0:1773941318.589124:0:2402:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000fcdea302. 00000040:00000010:0.0:1773941318.589129:0:2402:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 0000000028e8e696. 00000020:00000010:0.0:1773941318.589133:0:2402:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000cc36ab04. 00000020:00000010:0.0:1773941318.589136:0:2402:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000d2b6fbf4. 00000100:00000010:3.0:1773941318.589163:0:2384:0:(ptlrpcd.c:846:ptlrpcd_init()) kmalloced '(pd)': 392 at 00000000330805b3. 00000100:00000010:3.0:1773941318.589170:0:2384:0:(ptlrpcd.c:538:ptlrpcd_partners()) kmalloced '(pc->pc_partners)': 8 at 00000000e2b68c71. 00000100:00000010:3.0:1773941318.589172:0:2384:0:(ptlrpcd.c:538:ptlrpcd_partners()) kmalloced '(pc->pc_partners)': 8 at 000000009702c262. 00000100:00000010:2.0:1773941318.589320:0:2403:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 0000000095f52fcd. 00000020:00000010:2.0:1773941318.589326:0:2403:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000094c62faa. 00000020:00000010:2.0:1773941318.589328:0:2403:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000e7fab61a. 00000020:00000010:2.0:1773941318.589330:0:2403:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000525528a4. 00000040:00000010:2.0:1773941318.589332:0:2403:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000c2b007a7. 00000020:00000010:2.0:1773941318.589334:0:2403:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000ca2d154b. 00000020:00000010:2.0:1773941318.589336:0:2403:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000000c852add. 00000100:00000010:3.0:1773941318.589471:0:2404:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 00000000c22fc056. 00000020:00000010:3.0:1773941318.589476:0:2404:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000b2ba2461. 00000020:00000010:3.0:1773941318.589479:0:2404:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 0000000035a8af31. 00000020:00000010:3.0:1773941318.589481:0:2404:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000e1e411da. 00000040:00000010:3.0:1773941318.589484:0:2404:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000d1c6b71a. 00000020:00000010:3.0:1773941318.589487:0:2404:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000d054656d. 00000020:00000010:3.0:1773941318.589490:0:2404:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000f4878886. 00000100:00000010:1.0:1773941318.589540:0:2384:0:(connection.c:186:ptlrpc_connection_init()) kmalloced '(cpus_latency_qos)': 1088 at 000000002e3e341b. 00000100:00000010:1.0:1773941318.589703:0:2384:0:(nrs.c:1230:ptlrpc_nrs_policy_register()) kmalloced '(desc)': 72 at 00000000af193361. 00000100:00000010:1.0:1773941318.589707:0:2384:0:(nrs.c:1230:ptlrpc_nrs_policy_register()) kmalloced '(desc)': 72 at 000000000573d254. 00000100:00000010:1.0:1773941318.589708:0:2384:0:(nrs.c:1230:ptlrpc_nrs_policy_register()) kmalloced '(desc)': 72 at 00000000b9365d96. 00000100:00000010:1.0:1773941318.589710:0:2384:0:(nrs.c:1230:ptlrpc_nrs_policy_register()) kmalloced '(desc)': 72 at 0000000008f31b5e. 00000100:00000010:1.0:1773941318.589711:0:2384:0:(nrs.c:1230:ptlrpc_nrs_policy_register()) kmalloced '(desc)': 72 at 00000000ddb32ff0. 00000100:00000010:1.0:1773941318.589712:0:2384:0:(nrs.c:1230:ptlrpc_nrs_policy_register()) kmalloced '(desc)': 72 at 000000000dcbfe93. 00000001:00000010:1.0:1773941318.589844:0:2384:0:(nodemap_handler.c:4695:nodemap_config_alloc()) kmalloced '(config)': 480 at 00000000978e09d4. 00000001:00000010:1.0:1773941318.589848:0:2384:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at 00000000b3afa57b. 00000001:00000010:1.0:1773941318.589851:0:2384:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 8 at 000000009363637e. 00000001:00000010:1.0:1773941318.589854:0:2384:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 152 at 000000002e83cbb1. 00000001:00000010:1.0:1773941318.589861:0:2384:0:(nodemap_handler.c:3541:nodemap_create()) kmalloced '(nodemap)': 968 at 00000000d75bf1df. 00000001:00000010:1.0:1773941318.589891:0:2384:0:(nodemap_lproc.c:1443:lprocfs_nodemap_register()) kmalloced '(nm_entry)': 48 at 0000000034878de4. 02000000:00000010:0.0:1773941318.662481:0:2416:0:(upcall_cache.c:573:upcall_cache_init()) kmalloced '(cache)': 1208 at 00000000e67207b5. 02000000:00000010:0.0:1773941318.662492:0:2416:0:(upcall_cache.c:581:upcall_cache_init()) kmalloced '(cache->uc_hashtable)': 1024 at 000000004ef7e8da. 02000000:00000010:0.0:1773941318.662496:0:2416:0:(upcall_cache.c:573:upcall_cache_init()) kmalloced '(cache)': 1208 at 0000000047af91f5. 02000000:00000010:0.0:1773941318.662512:0:2416:0:(upcall_cache.c:581:upcall_cache_init()) kmalloced '(cache->uc_hashtable)': 16384 at 00000000d0e25214. 00000020:00000010:3.0:1773941320.406155:0:2468:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 000000005841cf01. 00000001:00000010:2.0:1773941320.489241:0:2481:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 00000000f10f3233. 00000100:00000010:2.0:1773941320.489274:0:2481:0:(client.c:689:ptlrpc_init_rq_pool()) kmalloced '(pool)': 56 at 00000000f625873f. 00000020:00000010:2.0:1773941320.489314:0:2481:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 00000000e32d0846. 00000020:00000010:2.0:1773941320.489338:0:2481:0:(lprocfs_status.c:118:ldebugfs_add_symlink()) kmalloced '(dest)': 129 at 0000000071a7d5d4. 00000020:00000010:2.0:1773941320.489350:0:2481:0:(lprocfs_status.c:128:ldebugfs_add_symlink()) kfreed 'dest': 129 at 0000000071a7d5d4. 00000020:00000010:3.0:1773941320.568129:0:2494:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 00000000b7971d29. 00000020:00000010:2.0:1773941320.647380:0:2507:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 0000000079e6ae25. 00000020:00000010:3.0:1773941320.702720:0:2521:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 00000000d51e30ed. 00008000:02000000:3.0:1773941320.756494:0:2534:0:(echo_client.c:2570:obdecho_init()) Echo OBD driver; http://www.lustre.org/ 00000020:00000010:3.0:1773941320.758704:0:2534:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 0000000038cefdd4. 00000020:00000010:3.0:1773941320.758925:0:2534:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 00000000bb91c352. 00000020:00000010:2.0:1773941320.945559:0:2557:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 00000000601e702c. 00000800:00000200:1.0:1773941321.269397:0:8:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_GOING_DOWN, iface=lo ifindex 1 state 0 ns 000000002ed8b4a5 00000800:00000200:1.0:1773941321.269677:0:8:0:(socklnd.c:2209:ksocknal_inet6addr_event()) addr6event: status NETDEV_DOWN, device lo, ip addr (einval), ns 000000002ed8b4a5 00000800:00000200:1.0:1773941321.269684:0:8:0:(socklnd.c:2112:ksocknal_handle_inetaddr_change()) Skipping device lo from namespace 000000002ed8b4a5 (expected 00000000209354f3) 00000800:00000200:1.0:1773941321.269692:0:8:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_DOWN, iface=lo ifindex 1 state 0 ns 000000002ed8b4a5 00000800:00000200:1.0:1773941321.269696:0:8:0:(socklnd.c:1972:ksocknal_handle_link_state_change()) Skipping device lo from namespace 000000002ed8b4a5 (expected 00000000209354f3) 00000800:00000200:1.0:1773941321.269788:0:8:0:(socklnd.c:2179:ksocknal_inetaddr_event()) addrevent: status NETDEV_DOWN device lo, ip addr 127.0.0.1, netmask 255.0.0.0 ns 000000002ed8b4a5. 00000800:00000200:1.0:1773941321.269793:0:8:0:(socklnd.c:2112:ksocknal_handle_inetaddr_change()) Skipping device lo from namespace 000000002ed8b4a5 (expected 00000000209354f3) 00000800:00000200:1.0:1773941321.269937:0:8:0:(socklnd.c:2153:ksocknal_device_event()) devevent: status=NETDEV_UNREGISTER, iface=lo ifindex 1 state 0 ns 000000002ed8b4a5 00000800:00000200:1.0:1773941321.269941:0:8:0:(socklnd.c:1972:ksocknal_handle_link_state_change()) Skipping device lo from namespace 000000002ed8b4a5 (expected 00000000209354f3) 00000020:00000010:2.0:1773941321.839158:0:2936:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 000000003dffa3bd. 00000020:00000010:2.0:1773941321.926305:0:2949:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 00000000567ebe31. 00000020:00000010:3.0:1773941322.048922:0:2962:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 00000000a7d37842. 00000020:00000010:3.0:1773941322.048952:0:2962:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 00000000bc05cd66. 00000020:00000010:2.0:1773941322.168226:0:2983:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 00000000fa54cf29. 00000020:00000010:2.0:1773941322.245849:0:2996:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 0000000058b22367. 00000020:00000010:2.0:1773941322.246115:0:2996:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 00000000d9092b02. 00000020:00000010:2.0:1773941322.327644:0:3011:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 000000003061c3f5. 00000020:00000010:2.0:1773941322.327663:0:3011:0:(genops.c:235:class_register_type()) kmalloced '(type)': 144 at 00000000a45f2c99. 00000020:00000010:1.0:1773941322.468177:0:3024:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at 00000000a0952e2c. 00000020:00000010:1.0:1773941322.468189:0:3024:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000009f617d8a. 00000020:00000010:1.0:1773941322.468193:0:3024:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000c2578391. 00000020:00000010:1.0:1773941322.468196:0:3024:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000434c6a58. 00000008:00000010:1.0:1773941322.468209:0:3024:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 000000009238d833. 00020000:00000010:1.0:1773941322.468214:0:3024:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 000000003ed3e558. 00008000:00000010:1.0:1773941322.468225:0:3024:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 000000008a33dfde. 00000080:00000010:1.0:1773941322.468236:0:3024:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 000000000e9b423a. 00000080:00000010:1.0:1773941322.468246:0:3024:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 000000006f9c95ae. 00000020:00000010:1.0:1773941322.468251:0:3024:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000ee68d43d. 00000008:00000010:1.0:1773941322.468263:0:3024:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at 00000000544bcf8c. 00020000:00000010:1.0:1773941322.468274:0:3024:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at 00000000c629c8f3. 00008000:00000010:1.0:1773941322.468290:0:3024:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at 00000000a7d448ce. 00000080:00000010:1.0:1773941322.468299:0:3024:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at 000000000d308395. 00000080:00000010:0.0:1773941370.999672:0:6296:0:(super25.c:253:lustre_init_fs_context()) kmalloced '(lmd)': 128 at 00000000b379851d. 00000020:00000010:0.0:1773941370.999692:0:6296:0:(obd_mount.c:1388:lustre_parse_monolithic()) kmalloced '(lmd->lmd_params)': 4096 at 0000000068441ade. 00000020:00000010:0.0:1773941370.999706:0:6296:0:(obd_mount.c:1590:lustre_parse_monolithic()) kmalloced '(lmd->lmd_dev)': 28 at 00000000116ca7ec. 00000020:00000010:0.0:1773941370.999711:0:6296:0:(obd_mount.c:1646:lustre_parse_monolithic()) kmalloced '(lmd->lmd_profile)': 14 at 00000000c03c35a6. 00000020:00000010:0.0:1773941370.999716:0:6296:0:(obd_mount.c:1674:lustre_parse_monolithic()) kmalloced '(lmd->lmd_opts)': 17 at 000000008a4f232a. 00000020:00000010:0.0:1773941370.999879:0:6296:0:(obd_mount.c:656:lustre_init_lsi()) kmalloced '(lsi)': 1880 at 00000000b6cdfaf7. 00000020:00000010:0.0:1773941370.999903:0:6296:0:(obd_mount.c:345:lustre_start_mgc()) kmalloced '(mgcname)': 23 at 000000007af8ad59. 00000020:00000010:0.0:1773941370.999907:0:6296:0:(obd_mount.c:353:lustre_start_mgc()) kmalloced '(data)': 192 at 00000000cd8776d0. 00000020:00000010:0.0:1773941370.999916:0:6296:0:(obd_mount.c:162:do_lcfg()) kmalloced '(lcfg)': 88 at 000000004d321bc6. 00000100:00000010:0.0:1773941370.999923:0:6296:0:(lustre_peer.c:71:class_add_uuid()) kmalloced '(data)': 704 at 0000000019da8332. 00000400:00000200:0.0:1773941370.999928:0:6296:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:0.0:1773941370.999933:0:6296:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.204.114@tcp 00000400:00000200:0.0:1773941370.999941:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941370.999956:0:6296:0:(peer.c:229:lnet_peer_alloc()) alloc '(lp)': 376 at 00000000614d8d0d. 00000400:00000200:0.0:1773941370.999982:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941370.999986:0:6296:0:(peer.c:262:lnet_peer_alloc()) 00000000614d8d0d nid 192.168.204.114@tcp 00000400:00000010:0.0:1773941370.999991:0:6296:0:(peer.c:193:lnet_peer_net_alloc()) alloc '(lpn)': 72 at 000000006312b4d5. 00000400:00000200:0.0:1773941370.999994:0:6296:0:(peer.c:202:lnet_peer_net_alloc()) 000000006312b4d5 net tcp 00000400:00000200:0.0:1773941370.999998:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941371.000001:0:6296:0:(peer.c:137:lnet_peer_ni_alloc()) alloc '(lpni)': 384 at 0000000051300511. 00000400:00000200:0.0:1773941371.000007:0:6296:0:(peer.c:183:lnet_peer_ni_alloc()) 0000000051300511 nid 192.168.204.114@tcp 00000400:00000200:0.0:1773941371.000014:0:6296:0:(peer.c:1091:lnet_peer_ni_clr_non_mr_pref_nid()) peer 192.168.204.114@tcp: -2 00000400:00000200:0.0:1773941371.000020:0:6296:0:(peer.c:1639:lnet_peer_attach_peer_ni()) peer 192.168.204.114@tcp NID 192.168.204.114@tcp flags 0x100001 00000400:00000200:0.0:1773941371.000026:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.000050:0:6296:0:(peer.c:2405:lnet_peer_queue_for_discovery()) Queue peer 192.168.204.114@tcp: 0 00000400:00000200:0.0:1773941371.000053:0:6296:0:(peer.c:2727:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:0.0:1773941371.000056:0:6296:0:(peer.c:2767:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:0.0:1773941371.000058:0:6296:0:(peer.c:2776:lnet_discover_peer_locked()) peer 192.168.204.114@tcp NID 192.168.204.114@tcp: 0. pending discovery 00000020:00000010:0.0:1773941371.000064:0:6296:0:(obd_mount.c:168:do_lcfg()) kfreed 'lcfg': 88 at 000000004d321bc6. 00000020:00000010:0.0:1773941371.000071:0:6296:0:(obd_mount.c:487:lustre_start_mgc()) kmalloced '(uuid)': 40 at 00000000acd85031. 00000020:00000010:0.0:1773941371.000080:0:6296:0:(obd_mount.c:162:do_lcfg()) kmalloced '(lcfg)': 120 at 00000000f64f1c48. 00000400:00000200:2.0:1773941371.000094:0:2399:0:(peer.c:3988:lnet_peer_discovery_wait_for_work()) woken: 0 00000020:00000010:0.0:1773941371.000106:0:6296:0:(genops.c:51:obd_device_alloc()) slab-alloced 'obd': 6584 at 00000000d340fc8c. 00000400:00000200:2.0:1773941371.000107:0:2399:0:(peer.c:4100:lnet_peer_discovery()) peer 192.168.204.114@tcp(00000000614d8d0d) state 0x100041 00000400:00000010:2.0:1773941371.000125:0:2399:0:(api-ni.c:1821:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at 000000008dba53fd. 00000020:00000010:0.0:1773941371.000126:0:6296:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at 00000000652ecc71. 00000400:00000010:2.0:1773941371.000134:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004cc288f2. 00000400:00000010:2.0:1773941371.000172:0:2399:0:(lib-lnet.h:478:lnet_rspt_alloc()) alloc 'rspt': 56 at 000000004147a294. 00000400:00000200:2.0:1773941371.000176:0:2399:0:(lib-move.c:5547:LNetGet()) LNetGet -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941371.000186:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941371.000195:0:2399:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000020:00000010:0.0:1773941371.000204:0:6296:0:(obd_mount.c:168:do_lcfg()) kfreed 'lcfg': 120 at 00000000f64f1c48. 00000400:00000200:2.0:1773941371.000205:0:2399:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941371.000214:0:2399:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000020:00000010:0.0:1773941371.000215:0:6296:0:(obd_mount.c:162:do_lcfg()) kmalloced '(lcfg)': 104 at 00000000f64f1c48. 00000400:00000200:2.0:1773941371.000220:0:2399:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941371.000225:0:2399:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:2.0:1773941371.000228:0:2399:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [1:1:256:4294967295] 192.168.204.114@tcp LPNI seq info [1:1:8:4294967295] 00000400:00000200:2.0:1773941371.000232:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941371.000238:0:2399:0:(lib-move.c:2009:lnet_handle_send()) rspt_next_hop_nid = 192.168.204.114@tcp 00000400:00000200:2.0:1773941371.000244:0:2399:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : GET try# 0 00000001:00000010:0.0:1773941371.000246:0:6296:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at 000000005113a6fa. 00000800:00000200:2.0:1773941371.000253:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.204.114@tcp 00000001:00000010:0.0:1773941371.000256:0:6296:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at 00000000f1656fdc. 00000800:00000010:2.0:1773941371.000257:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at 00000000172f8ab3. 00000001:00000010:0.0:1773941371.000267:0:6296:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 000000006390162c. 00000400:00000200:2.0:1773941371.000272:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000010:2.0:1773941371.000277:0:2399:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at 000000006663d74c. 00000800:00000010:2.0:1773941371.000280:0:2399:0:(socklnd.c:106:ksocknal_create_conn_cb()) alloc '(conn_cb)': 192 at 00000000cbdcd5aa. 00000001:00000010:0.0:1773941371.000281:0:6296:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 000000009503ca68. 00000800:00000200:2.0:1773941371.000288:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006663d74c] -> 12345-192.168.204.114@tcp (2) 00000001:00000010:0.0:1773941371.000290:0:6296:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000e394c09c. 00000800:00000200:2.1F:1773941371.000297:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006663d74c] -> 12345-192.168.204.114@tcp (2) 00000001:00000010:0.0:1773941371.000301:0:6296:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000c4297132. 00000020:00000010:0.0:1773941371.000310:0:6296:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004ff72262. 00000020:00000010:0.0:1773941371.000318:0:6296:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 00000000cbfe9690. 00000400:00000200:2.0:1773941371.000322:0:2399:0:(peer.c:3788:lnet_peer_send_ping()) peer 192.168.204.114@tcp 00000400:00000200:2.0:1773941371.000325:0:2399:0:(peer.c:4118:lnet_peer_discovery()) peer 192.168.204.114@tcp(00000000614d8d0d) state 0x100241 rc 0 00000020:00000010:0.0:1773941371.000334:0:6296:0:(tgt_main.c:756:tgt_ses_key_init()) slab-alloced 'session': 200 at 00000000211306bd. 00000008:00000010:0.0:1773941371.000354:0:6296:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at 0000000020073e0c. 00020000:00000010:0.0:1773941371.000372:0:6296:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at 00000000e37deccf. 00008000:00000010:0.0:1773941371.000392:0:6296:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at 000000007cfd2609. 00000080:00000010:0.0:1773941371.000412:0:6296:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at 0000000070a0e965. 00000020:00000010:0.0:1773941371.000418:0:6296:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000b6d0fb77. 00000020:00000010:0.0:1773941371.000423:0:6296:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000b2ef6fef. 00000040:00000010:0.0:1773941371.000429:0:6296:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000fb0d7ffe. 00000020:00000010:0.0:1773941371.000435:0:6296:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 0000000003d81041. 00000020:00000010:0.0:1773941371.000440:0:6296:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 000000006b47a18f. 00040000:00000010:0.0:1773941371.000445:0:6296:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 0000000072edce6b. 00040000:00000010:0.0:1773941371.000449:0:6296:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000c7ea8da2. 10000000:00000010:0.0:1773941371.000454:0:6296:0:(mgc_request.c:852:mgc_device_alloc()) kmalloced '(lu)': 64 at 000000007993ff0f. 00010000:00000010:0.0:1773941371.000465:0:6296:0:(ldlm_lib.c:421:client_obd_setup()) kmalloced '(cli->cl_read_io_latency_by_size)': 3920 at 000000008195e776. 00010000:00000010:0.0:1773941371.000472:0:6296:0:(ldlm_lib.c:434:client_obd_setup()) kmalloced '(cli->cl_write_io_latency_by_size)': 3920 at 000000003594877b. 00010000:00000010:0.0:1773941371.000481:0:6296:0:(ldlm_lockd.c:3356:ldlm_setup()) kmalloced '(ldlm_state)': 32 at 00000000859fda21. 00000100:00000010:0.0:1773941371.000586:0:6296:0:(service.c:745:ptlrpc_register_service()) kmalloced '(service)': 360 at 000000003038117a. 00000100:00000010:0.0:1773941371.000595:0:6296:0:(service.c:803:ptlrpc_register_service()) kmalloced '(svcpt)': 784 at 000000007a0c7935. 00000100:00000010:0.0:1773941371.000612:0:6296:0:(service.c:627:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_array)': 12080 at 000000008a6cb41b. 00000100:00000010:0.0:1773941371.000622:0:6296:0:(service.c:635:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_count)': 3020 at 00000000bad5b02e. 00000100:00000010:0.0:1773941371.000628:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000009d9499af. 00000100:00000010:0.0:1773941371.000646:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000006e5e619c. 00000100:00000010:0.0:1773941371.000651:0:6296:0:(service.c:803:ptlrpc_register_service()) kmalloced '(svcpt)': 784 at 00000000ab04a35f. 00000100:00000010:0.0:1773941371.000665:0:6296:0:(service.c:627:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_array)': 12080 at 00000000f4dfff06. 00000100:00000010:0.0:1773941371.000672:0:6296:0:(service.c:635:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_count)': 3020 at 0000000066321ebb. 00000100:00000010:0.0:1773941371.000677:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000fd10c219. 00000100:00000010:0.0:1773941371.000681:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000dba80c16. 00000400:00000200:0.0:1773941371.000686:0:6296:0:(lib-ptl.c:921:LNetSetLazyPortal()) Setting portal 15 lazy 00000020:00000010:0.0:1773941371.000729:0:6296:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000bf82a7c6. 00000020:00000010:0.0:1773941371.000735:0:6296:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 3680 at 000000001315210c. 00000100:00000010:0.0:1773941371.000786:0:6296:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 00000000a049415e. 00000100:00000010:0.0:1773941371.000793:0:6296:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at 00000000b5b53ac5. 00000100:00000010:0.0:1773941371.000796:0:6296:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 00000000ff4966a1. 00000100:00000010:0.0:1773941371.000799:0:6296:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 000000007bb95a35. 00000100:00000010:0.0:1773941371.000801:0:6296:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 0000000027124f3b. 00000100:00000010:0.0:1773941371.000809:0:6296:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 0000000083e17e07. 00000100:00000010:0.0:1773941371.000811:0:6296:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at 000000007e9c2306. 00000100:00000010:0.0:1773941371.000820:0:6296:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 0000000016bc1d9b. 00000100:00000010:0.0:1773941371.000822:0:6296:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 00000000ea18f74d. 00000100:00000010:0.0:1773941371.000823:0:6296:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 000000005e1469b5. 00000100:00000200:0.0:1773941371.000856:0:6296:0:(service.c:848:ptlrpc_register_service()) ldlm_cbd: Started, listening on portal 15 00000100:00000010:0.0:1773941371.000861:0:6296:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at 000000001d35575b. 00000800:00000010:2.0:1773941371.001276:0:2388:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4712 at 0000000002d6902f. 00000800:00000010:2.0:1773941371.001281:0:2388:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 00000000562bd131. 00000800:00000010:2.0:1773941371.001288:0:2388:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 000000008ed05b9e. 00000800:00000010:2.0:1773941371.001299:0:2388:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 000000008ed05b9e. 00000100:00000010:0.0:1773941371.001382:0:6297:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at 000000002033e4be. 00000020:00000010:0.0:1773941371.001389:0:6297:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 000000003075bbce. 00000020:00000010:0.0:1773941371.001397:0:6297:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000480cfa63. 00000020:00000010:0.0:1773941371.001401:0:6297:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000004d40d50c. 00000040:00000010:0.0:1773941371.001405:0:6297:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 0000000096a13531. 00000020:00000010:0.0:1773941371.001408:0:6297:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 000000009c59969e. 00000020:00000010:0.0:1773941371.001426:0:6297:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at 00000000784688e3. 00000020:00000010:0.0:1773941371.001431:0:6297:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000cfb5b037. 80000000:00000010:0.0:1773941371.001435:0:6297:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000e8cddd88. 40000000:00000010:0.0:1773941371.001439:0:6297:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 00000000e161c05c. 00100000:00000010:0.0:1773941371.001444:0:6297:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 00000000b8960919. 00040000:00000010:0.0:1773941371.001447:0:6297:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 000000008633ea2a. 00040000:00000010:0.0:1773941371.001451:0:6297:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 000000007973a4e4. 00040000:00000010:0.0:1773941371.001455:0:6297:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 0000000099858687. 00000004:00000010:0.0:1773941371.001459:0:6297:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 0000000007f89e2a. 00000004:00000010:0.0:1773941371.001465:0:6297:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at 000000008f413fda. 00000004:00000010:0.0:1773941371.001470:0:6297:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 00000000a95dea61. 00002000:00000010:0.0:1773941371.001475:0:6297:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 00000000521c56e7. 00000004:00000010:0.0:1773941371.001480:0:6297:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at 00000000ada97eae. 00000100:00000200:0.0:1773941371.001485:0:6297:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_cbd: registering portal 15 00000400:00000010:0.0:1773941371.001492:0:6297:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000009ff475a. 00000400:00000010:0.0:1773941371.001498:0:6297:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000046eee8a8. 00000100:00000010:0.0:1773941371.001509:0:6297:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at 00000000f1d5ff64. 00000100:00000200:0.0:1773941371.001536:0:6297:0:(service.c:3086:ptlrpc_main()) service thread 0 (#1) started 00000100:00000010:2.0:1773941371.001598:0:6296:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at 00000000583dc2e9. 00000100:00000010:0.0:1773941371.001859:0:6298:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at 0000000044a4d55d. 00000020:00000010:0.0:1773941371.001864:0:6298:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 000000006207c8ca. 00000020:00000010:0.0:1773941371.001871:0:6298:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000a2a11f06. 00000020:00000010:0.0:1773941371.001881:0:6298:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 0000000054bea8b9. 00000040:00000010:0.0:1773941371.001885:0:6298:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 0000000028ffb1e0. 00000020:00000010:0.0:1773941371.001889:0:6298:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 0000000023bb7ac6. 00000400:00000200:3.0:1773941371.001895:0:2388:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000020:00000010:0.0:1773941371.001898:0:6298:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at 000000001f46179c. 00000800:00000200:3.1F:1773941371.001901:0:2388:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 1, ksnr_connected 2 ksnr_max_conns 1 00000020:00000010:0.0:1773941371.001904:0:6298:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000276ae665. 80000000:00000010:0.0:1773941371.001909:0:6298:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 0000000013e94cae. 40000000:00000010:0.0:1773941371.001921:0:6298:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 00000000eaf059cf. 00000800:00000200:3.1:1773941371.001927:0:2388:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00100000:00000010:0.0:1773941371.001929:0:6298:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 00000000583e30f9. 00000800:00000200:3.1:1773941371.001932:0:2388:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 00000000172f8ab3 type 2, nob 96 niov 1 nkiov 0 00040000:00000010:0.0:1773941371.001938:0:6298:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 000000007c0c30a3. 00000800:00000200:3.0:1773941371.001943:0:2388:0:(socklnd.c:1183:ksocknal_create_conn()) New conn 12345-192.168.204.114@tcp p 3.x 192.168.204.14 -> 192.168.204.114:988 incarnation:1773941325747225435 sched[0] 00000800:00000010:3.0:1773941371.001950:0:2388:0:(socklnd.c:1191:ksocknal_create_conn()) kfreed 'hello': 144 at 00000000562bd131. 00040000:00000010:0.0:1773941371.001953:0:6298:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 0000000004e74add. 00040000:00000010:0.0:1773941371.001960:0:6298:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 0000000052878b6c. 00000400:00000200:3.0:1773941371.002048:0:2388:0:(lib-socket.c:453:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.204.114:988 00000004:00000010:0.0:1773941371.002074:0:6298:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 000000001a59f416. 00000004:00000010:0.0:1773941371.002085:0:6298:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at 000000009fe20e04. 00000004:00000010:0.0:1773941371.002091:0:6298:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 000000002b3d7691. 00002000:00000010:0.0:1773941371.002096:0:6298:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 0000000059bd7a2f. 00000004:00000010:0.0:1773941371.002101:0:6298:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at 0000000019658bdc. 00000100:00000010:0.0:1773941371.002108:0:6298:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at 00000000ed48b220. 00000800:00000200:1.0:1773941371.002126:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000200:0.0:1773941371.002130:0:6298:0:(service.c:3086:ptlrpc_main()) service thread 1 (#2) started 00000800:00000010:1.0:1773941371.002131:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at 00000000172f8ab3. 00000400:00000200:1.0:1773941371.002138:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941371.002146:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: GET: OK 00000400:00000200:1.0:1773941371.002153:0:2393:0:(peer.c:3081:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1773941371.002158:0:2393:0:(peer.c:3033:lnet_discovery_event_send()) Ping Send to 192.168.204.114@tcp: 0 00000100:00000010:2.0:1773941371.002164:0:6296:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at 0000000091036798. 00000100:00000010:2.0:1773941371.002421:0:6299:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at 00000000246037ad. 00000020:00000010:2.0:1773941371.002428:0:6299:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 0000000007562f3a. 00000020:00000010:2.0:1773941371.002436:0:6299:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000003d3cbf21. 00000020:00000010:2.0:1773941371.002441:0:6299:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000008e37e2c7. 00000040:00000010:2.0:1773941371.002445:0:6299:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000307beb49. 00000020:00000010:2.0:1773941371.002450:0:6299:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000626b79ae. 00000020:00000010:2.0:1773941371.002469:0:6299:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at 000000002cc1c5dd. 00000020:00000010:2.0:1773941371.002476:0:6299:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 000000006ca81ac1. 80000000:00000010:2.0:1773941371.002481:0:6299:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000d753596d. 40000000:00000010:2.0:1773941371.002491:0:6299:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 00000000064b1bf8. 00100000:00000010:2.0:1773941371.002497:0:6299:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 000000005d40785e. 00040000:00000010:2.0:1773941371.002503:0:6299:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 00000000aa66cc9a. 00040000:00000010:2.0:1773941371.002508:0:6299:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 0000000097b5ebe3. 00040000:00000010:2.0:1773941371.002512:0:6299:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000856196e7. 00000004:00000010:2.0:1773941371.002518:0:6299:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 00000000acd86617. 00000400:00000200:0.0:1773941371.002523:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000004:00000010:2.0:1773941371.002530:0:6299:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at 00000000e766ee01. 00000400:00000200:0.0:1773941371.002530:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : REPLY - for me 00000004:00000010:2.0:1773941371.002540:0:6299:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 000000002db94588. 00002000:00000010:2.0:1773941371.002545:0:6299:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 0000000084a98cca. 00000800:00000010:3.0:1773941371.002548:0:2388:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4712 at 00000000ddd7a00c. 00000004:00000010:2.0:1773941371.002551:0:6299:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at 00000000511cdfb0. 00000400:00000200:0.0:1773941371.002558:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000010:3.0:1773941371.002560:0:2388:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 000000002a610397. 00000800:00000010:3.0:1773941371.002568:0:2388:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 00000000b9445e27. 00000100:00000200:2.0:1773941371.002568:0:6299:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_cbd: registering portal 15 00000400:00000200:0.0:1773941371.002569:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941371.002580:0:6299:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000453cc0fa. 00000800:00000010:3.0:1773941371.002583:0:2388:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 00000000b9445e27. 00000400:00000200:0.0:1773941371.002591:0:2394:0:(lib-move.c:4647:lnet_parse_reply()) 192.168.204.14@tcp: Reply from 12345-192.168.204.114@tcp of length 48/48 into md 0x5 00000400:00000010:2.0:1773941371.002601:0:6299:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000b88670a8. 00000400:00000200:0.0:1773941371.002613:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000010:2.0:1773941371.002615:0:6299:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at 000000000509c806. 00000400:00000200:0.0:1773941371.002623:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: REPLY: OK 00000400:00000200:0.0:1773941371.002630:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004cc288f2 00000400:00000010:0.0:1773941371.002643:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 000000004cc288f2. 00000400:00000200:0.0:1773941371.002648:0:2394:0:(peer.c:3081:lnet_discovery_event_handler()) Received event: 3 00000400:00000200:0.0:1773941371.002654:0:2394:0:(peer.c:2911:lnet_discovery_event_reply()) Peer 192.168.204.114@tcp has discovery enabled 00000400:00000200:0.0:1773941371.002657:0:2394:0:(peer.c:2923:lnet_discovery_event_reply()) peer 192.168.204.114@tcp(00000000614d8d0d) is MR 00000100:00000010:3.0:1773941371.002658:0:6296:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at 00000000b0b24826. 00000100:00000200:2.0:1773941371.002658:0:6299:0:(service.c:3086:ptlrpc_main()) service thread 0 (#1) started 00000400:00000200:0.0:1773941371.002669:0:2394:0:(peer.c:2992:lnet_discovery_event_reply()) peer 192.168.204.114@tcp data present 363512030. state = 0x100241 00000400:00000200:0.0:1773941371.002676:0:2394:0:(peer.c:2405:lnet_peer_queue_for_discovery()) Queue peer 192.168.204.114@tcp: -114 00000400:00000200:2.0:1773941371.002699:0:2399:0:(peer.c:3988:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:2.0:1773941371.002708:0:2399:0:(peer.c:4100:lnet_peer_discovery()) peer 192.168.204.114@tcp(00000000614d8d0d) state 0x1000c1 00000400:00000010:2.0:1773941371.002718:0:2399:0:(peer.c:3265:lnet_peer_merge_data()) alloc '(curnis)': 40 at 00000000da8759c5. 00000400:00000010:2.0:1773941371.002722:0:2399:0:(peer.c:3266:lnet_peer_merge_data()) alloc '(addnis)': 48 at 00000000f15016ec. 00000400:00000010:2.0:1773941371.002724:0:2399:0:(peer.c:3267:lnet_peer_merge_data()) alloc '(delnis)': 40 at 00000000fe8185ce. 00000400:00000200:2.0:1773941371.002731:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941371.002736:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941371.002738:0:2399:0:(peer.c:3405:lnet_peer_merge_data()) kfreed 'curnis': 40 at 00000000da8759c5. 00000400:00000010:2.0:1773941371.002742:0:2399:0:(peer.c:3406:lnet_peer_merge_data()) kfreed 'addnis': 48 at 00000000f15016ec. 00000400:00000010:2.0:1773941371.002747:0:2399:0:(peer.c:3407:lnet_peer_merge_data()) kfreed 'delnis': 40 at 00000000fe8185ce. 00000400:00000010:2.0:1773941371.002750:0:2399:0:(api-ni.c:1839:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at 000000008dba53fd. 00000400:00000200:2.0:1773941371.002759:0:2399:0:(peer.c:3410:lnet_peer_merge_data()) peer 192.168.204.114@tcp (00000000614d8d0d): 0 00000400:00000200:2.0:1773941371.002766:0:2399:0:(peer.c:3714:lnet_peer_data_present()) peer 192.168.204.114@tcp(00000000614d8d0d): 0. state = 0x100141 00000400:00000200:2.0:1773941371.002771:0:2399:0:(peer.c:4118:lnet_peer_discovery()) peer 192.168.204.114@tcp(00000000614d8d0d) state 0x100141 rc 1 00000400:00000200:2.0:1773941371.002778:0:2399:0:(peer.c:4100:lnet_peer_discovery()) peer 192.168.204.114@tcp(00000000614d8d0d) state 0x100141 00000400:00000010:2.0:1773941371.002784:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000234c2bb5. 00000400:00000010:2.0:1773941371.002793:0:2399:0:(lib-lnet.h:478:lnet_rspt_alloc()) alloc 'rspt': 56 at 000000005298ce59. 00000400:00000200:2.0:1773941371.002800:0:2399:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941371.002809:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941371.002824:0:2399:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source Specified: 192.168.204.14@tcp to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941371.002830:0:2399:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [2:2:256:4294967295] 192.168.204.114@tcp LPNI seq info [2:2:8:4294967295] 00000400:00000200:2.0:1773941371.002834:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941371.002839:0:2399:0:(lib-move.c:2009:lnet_handle_send()) rspt_next_hop_nid = 192.168.204.114@tcp 00000400:00000200:2.0:1773941371.002845:0:2399:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:192.168.204.14@tcp) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:2.0:1773941371.002851:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:2.0:1773941371.002855:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000172f8ab3. 00000800:00000200:2.0:1773941371.002860:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006663d74c] -> 12345-192.168.204.114@tcp (3) 00000800:00000200:2.0:1773941371.002866:0:2399:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:2.0:1773941371.002869:0:2399:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 00000000172f8ab3 type 1, nob 144 niov 1 nkiov 1 00000400:00000200:2.0:1773941371.002887:0:2399:0:(peer.c:3922:lnet_peer_send_push()) peer 192.168.204.114@tcp 00000400:00000200:2.0:1773941371.002891:0:2399:0:(peer.c:4118:lnet_peer_discovery()) peer 192.168.204.114@tcp(00000000614d8d0d) state 0x100541 rc 0 00000100:00000010:2.0:1773941371.002941:0:6300:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at 00000000fe8185ce. 00000020:00000010:2.0:1773941371.002946:0:6300:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 00000000f15016ec. 00000020:00000010:2.0:1773941371.002952:0:6300:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000008dba53fd. 00000800:00000200:1.0:1773941371.002955:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000020:00000010:2.0:1773941371.002957:0:6300:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000abed6f77. 00000800:00000010:1.0:1773941371.002959:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000172f8ab3. 00000400:00000200:1.0:1773941371.002989:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941371.002995:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941371.002998:0:2393:0:(peer.c:3081:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:1.0:1773941371.003004:0:2393:0:(peer.c:3033:lnet_discovery_event_send()) Push Send to 192.168.204.114@tcp: 0 00000040:00000010:2.0:1773941371.003005:0:6300:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000149096a2. 00000020:00000010:2.0:1773941371.003011:0:6300:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000c7b23d2f. 00000020:00000010:2.0:1773941371.003018:0:6300:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at 000000000de9f25b. 00000020:00000010:2.0:1773941371.003022:0:6300:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000b656f5d9. 80000000:00000010:2.0:1773941371.003026:0:6300:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000b84eab15. 40000000:00000010:2.0:1773941371.003028:0:6300:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 00000000da8759c5. 00100000:00000010:2.0:1773941371.003032:0:6300:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 000000003c838ca4. 00040000:00000010:2.0:1773941371.003035:0:6300:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 000000008fb48073. 00040000:00000010:2.0:1773941371.003039:0:6300:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 00000000f1409975. 00040000:00000010:2.0:1773941371.003042:0:6300:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 0000000037a8d18e. 00000004:00000010:2.0:1773941371.003046:0:6300:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 0000000003a796f7. 00000004:00000010:2.0:1773941371.003050:0:6300:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at 00000000ab3a57c3. 00000004:00000010:2.0:1773941371.003065:0:6300:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 000000001c51f1ee. 00002000:00000010:2.0:1773941371.003069:0:6300:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 000000002b2ee536. 00000004:00000010:2.0:1773941371.003073:0:6300:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at 00000000adc540bc. 00000100:00000010:2.0:1773941371.003082:0:6300:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at 0000000042b43fe0. 00000100:00000200:2.0:1773941371.003098:0:6300:0:(service.c:3086:ptlrpc_main()) service thread 1 (#2) started 00000400:00000200:0.0:1773941371.003184:0:2388:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:0.1F:1773941371.003194:0:2388:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 2, ksnr_connected 6 ksnr_max_conns 1 00000800:00000200:0.0:1773941371.003203:0:2388:0:(socklnd.c:1183:ksocknal_create_conn()) New conn 12345-192.168.204.114@tcp p 3.x 192.168.204.14 -> 192.168.204.114:988 incarnation:1773941325747225435 sched[0] 00000800:00000010:0.0:1773941371.003206:0:2388:0:(socklnd.c:1191:ksocknal_create_conn()) kfreed 'hello': 144 at 000000002a610397. 00000100:00000010:3.0:1773941371.003239:0:6296:0:(service.c:745:ptlrpc_register_service()) kmalloced '(service)': 360 at 00000000404006ad. 00000100:00000010:3.0:1773941371.003249:0:6296:0:(service.c:803:ptlrpc_register_service()) kmalloced '(svcpt)': 784 at 00000000aaa05c30. 00000100:00000010:3.0:1773941371.003263:0:6296:0:(service.c:627:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_array)': 12080 at 00000000cfc3fcbe. 00000100:00000010:3.0:1773941371.003271:0:6296:0:(service.c:635:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_count)': 3020 at 00000000ddb939fd. 00000100:00000010:3.0:1773941371.003275:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000008f8bf565. 00000400:00000200:0.0:1773941371.003279:0:2388:0:(lib-socket.c:453:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.204.114:988 00000100:00000010:3.0:1773941371.003280:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000419e6687. 00000100:00000010:3.0:1773941371.003285:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000015c7f83f. 00000100:00000010:3.0:1773941371.003292:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000866fc449. 00000100:00000010:3.0:1773941371.003294:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000630ce54d. 00000100:00000010:3.0:1773941371.003298:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000002b4bf3eb. 00000100:00000010:3.0:1773941371.003301:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000003aae3a6d. 00000400:00000200:0.0:1773941371.003341:0:2388:0:(lib-socket.c:453:lnet_sock_connect()) Error -99 connecting 1022 -> 192.168.204.114:988 00000100:00000010:3.0:1773941371.003375:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000007526ebcf. 00000100:00000010:3.0:1773941371.003412:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000009d1acdeb. 00000400:00000200:1.0:1773941371.003419:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000010:3.0:1773941371.003421:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000e6c1e8db. 00000100:00000010:3.0:1773941371.003428:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000da45d957. 00000100:00000010:3.0:1773941371.003435:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000094b14934. 00000400:00000200:1.0:1773941371.003437:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : ACK - for me 00000400:00000200:1.0:1773941371.003465:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.003470:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.003477:0:2393:0:(lib-move.c:4700:lnet_parse_ack()) 192.168.204.14@tcp: ACK from 12345-192.168.204.114@tcp into md 0x15 00000400:00000200:1.0:1773941371.003482:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941371.003485:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: ACK: OK 00000400:00000200:1.0:1773941371.003496:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000234c2bb5 00000400:00000010:1.0:1773941371.003499:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000234c2bb5. 00000400:00000200:1.0:1773941371.003502:0:2393:0:(peer.c:3081:lnet_discovery_event_handler()) Received event: 4 00000400:00000200:1.0:1773941371.003506:0:2393:0:(peer.c:2799:lnet_discovery_event_ack()) peer 192.168.204.114@tcp ev->status 0 00000400:00000200:1.0:1773941371.003510:0:2393:0:(peer.c:2405:lnet_peer_queue_for_discovery()) Queue peer 192.168.204.114@tcp: -114 00000100:00000010:3.0:1773941371.003523:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000a6e890af. 00000100:00000010:3.0:1773941371.003529:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000443ab54e. 00000100:00000010:3.0:1773941371.003532:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000039985f3e. 00000400:00000200:1.0:1773941371.003539:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000010:3.0:1773941371.003544:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000007ac0e6d8. 00000400:00000200:1.0:1773941371.003552:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000100:00000010:3.0:1773941371.003553:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000424b19b8. 00000400:00000200:2.0:1773941371.003558:0:2399:0:(peer.c:3988:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:1.0:1773941371.003564:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000010:3.0:1773941371.003570:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000a9d472d1. 00000400:00000200:1.0:1773941371.003574:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000010:3.0:1773941371.003576:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000009c54427d. 00000100:00000010:3.0:1773941371.003581:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000d7df8bd4. 00000400:00000200:1.0:1773941371.003582:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000010:3.0:1773941371.003584:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000b9bd4608. 00000400:00000200:2.0:1773941371.003589:0:2399:0:(peer.c:4100:lnet_peer_discovery()) peer 192.168.204.114@tcp(00000000614d8d0d) state 0x100141 00000100:00000010:3.0:1773941371.003592:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000098d63316. 00000400:00000200:1.0:1773941371.003601:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 48 into portal 0 MB=0x8000000000000000 00000100:00000010:3.0:1773941371.003602:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000fa8fd783. 00000400:00000200:2.0:1773941371.003603:0:2399:0:(peer.c:3844:lnet_peer_discovered()) peer 192.168.204.114@tcp 00000400:00000200:2.0:1773941371.003614:0:2399:0:(peer.c:4118:lnet_peer_discovery()) peer 192.168.204.114@tcp(00000000614d8d0d) state 0x100111 rc 0 00000100:00000010:3.0:1773941371.003615:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000004b62387d. 00000100:00000010:3.0:1773941371.003620:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000612b0046. 00000400:00000200:2.0:1773941371.003626:0:2399:0:(peer.c:2421:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 192.168.204.114@tcp 00000400:00000200:1.0:1773941371.003628:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 0 from 12345-192.168.204.114@tcp of length 48/48 into md 0x9 [1] + 0 00000100:00000010:3.0:1773941371.003629:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000d9fd9544. 00000100:00000010:3.0:1773941371.003647:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000006a178bec. 00000400:00000010:1.0:1773941371.003650:0:2393:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 0000000025f467c2. 00000100:00000010:3.0:1773941371.003654:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000002c93222e. 00000100:00000010:3.0:1773941371.003658:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000052b71d78. 00000400:00000200:1.0:1773941371.003658:0:2393:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 000000006d307180 00000400:00000200:1.0:1773941371.003697:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941371.003703:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941371.003706:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006d307180 00000400:00000010:1.0:1773941371.003708:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 000000006d307180. 00000400:00000200:1.0:1773941371.003711:0:2393:0:(api-ni.c:2341:lnet_push_target_event_handler()) type 2 status 0 unlinked 1 00000400:00000200:1.0:1773941371.003715:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.003718:0:2393:0:(peer.c:2570:lnet_peer_push_event()) peer 192.168.204.114@tcp(00000000614d8d0d) is MR 00000400:00000010:1.0:1773941371.003722:0:2393:0:(api-ni.c:1821:lnet_ping_buffer_alloc()) alloc '(pbuf)': 57 at 000000001e4421df. 00000400:00000200:1.0:1773941371.003726:0:2393:0:(peer.c:2644:lnet_peer_push_event()) Received Push 192.168.204.114@tcp 363512030 00000400:00000200:1.0:1773941371.003742:0:2393:0:(peer.c:2405:lnet_peer_queue_for_discovery()) Queue peer 192.168.204.114@tcp: 0 00000400:00000200:1.0:1773941371.003748:0:2393:0:(lib-md.c:96:lnet_cpt_of_md()) Couldn't resolve first page of md 0000000000000000 with offset 0 00000400:00000200:1.0:1773941371.003752:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.003758:0:2393:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source Specified: 192.168.204.14@tcp to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941371.003765:0:2393:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [3:3:256:4294967295] 192.168.204.114@tcp LPNI seq info [3:3:8:4294967295] 00000400:00000200:1.0:1773941371.003772:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941371.003774:0:2399:0:(peer.c:3988:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000010:2.0:1773941371.003783:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000004d11e96d. 00000100:00000010:3.0:1773941371.003792:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000dafa4d4d. 00000400:00000010:2.0:1773941371.003799:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000bda4546c. 00000100:00000010:3.0:1773941371.003802:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000bfa293c6. 00000400:00000200:1.0:1773941371.003803:0:2393:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:192.168.204.14@tcp) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) 192.168.204.114@tcp : ACK try# 0 00000800:00000010:0.0:1773941371.003804:0:2388:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4712 at 00000000f086b99b. 00000400:00000200:2.0:1773941371.003811:0:2399:0:(api-ni.c:2331:lnet_push_target_post()) posted push target 000000005759e74f 00000800:00000010:0.0:1773941371.003815:0:2388:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at 00000000f1ad2f8e. 00000100:00000010:3.0:1773941371.003818:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000cccf1258. 00000800:00000010:0.0:1773941371.003828:0:2388:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at 00000000d3cdc50b. 00000400:00000200:2.0:1773941371.003833:0:2399:0:(peer.c:4100:lnet_peer_discovery()) peer 192.168.204.114@tcp(00000000614d8d0d) state 0x1000d1 00000800:00000200:1.0:1773941371.003834:0:2393:0:(socklnd_cb.c:1001:ksocknal_send()) sending 0 bytes in 1 frags to 12345-192.168.204.114@tcp 00000400:00000010:2.0:1773941371.003844:0:2399:0:(peer.c:3265:lnet_peer_merge_data()) alloc '(curnis)': 40 at 00000000109b891b. 00000800:00000010:0.0:1773941371.003847:0:2388:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at 00000000d3cdc50b. 00000800:00000010:1.0:1773941371.003849:0:2393:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000002a8b63bd. 00000400:00000010:2.0:1773941371.003852:0:2399:0:(peer.c:3266:lnet_peer_merge_data()) alloc '(addnis)': 48 at 00000000b29584b9. 00000100:00000010:3.0:1773941371.003855:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000b6e41cf7. 00000400:00000010:2.0:1773941371.003859:0:2399:0:(peer.c:3267:lnet_peer_merge_data()) alloc '(delnis)': 40 at 000000006cce1d56. 00000100:00000010:3.0:1773941371.003861:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000d549c684. 00000800:00000200:1.0:1773941371.003866:0:2393:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006663d74c] -> 12345-192.168.204.114@tcp (4) 00000100:00000010:3.0:1773941371.003868:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000096666be6. 00000400:00000200:2.0:1773941371.003876:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000010:3.0:1773941371.003879:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000549ae169. 00000100:00000010:3.0:1773941371.003887:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000c6fb7c07. 00000800:00000200:1.0:1773941371.003889:0:2393:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000400:00000200:2.0:1773941371.003893:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941371.003901:0:2399:0:(peer.c:3405:lnet_peer_merge_data()) kfreed 'curnis': 40 at 00000000109b891b. 00000800:00000200:1.0:1773941371.003901:0:2393:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 000000002a8b63bd type 0, nob 96 niov 1 nkiov 0 00000100:00000010:3.0:1773941371.003902:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000089cae324. 00000100:00000010:3.0:1773941371.003916:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000ed9f1bc7. 00000400:00000010:2.0:1773941371.003916:0:2399:0:(peer.c:3406:lnet_peer_merge_data()) kfreed 'addnis': 48 at 00000000b29584b9. 00000400:00000010:2.0:1773941371.003923:0:2399:0:(peer.c:3407:lnet_peer_merge_data()) kfreed 'delnis': 40 at 000000006cce1d56. 00000400:00000010:2.0:1773941371.003928:0:2399:0:(api-ni.c:1839:lnet_ping_buffer_free()) kfreed 'pbuf': 57 at 000000001e4421df. 00000100:00000010:3.0:1773941371.003934:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000e475fa1e. 00000400:00000200:2.0:1773941371.003934:0:2399:0:(peer.c:3410:lnet_peer_merge_data()) peer 192.168.204.114@tcp (00000000614d8d0d): 0 00000400:00000200:2.0:1773941371.003941:0:2399:0:(peer.c:3714:lnet_peer_data_present()) peer 192.168.204.114@tcp(00000000614d8d0d): 0. state = 0x100151 00000100:00000010:3.0:1773941371.003943:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000087dde907. 00000400:00000200:2.0:1773941371.003948:0:2399:0:(peer.c:4118:lnet_peer_discovery()) peer 192.168.204.114@tcp(00000000614d8d0d) state 0x100151 rc 1 00000100:00000010:3.0:1773941371.003950:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000099603f39. 00000400:00000200:2.0:1773941371.003955:0:2399:0:(peer.c:4100:lnet_peer_discovery()) peer 192.168.204.114@tcp(00000000614d8d0d) state 0x100151 00000100:00000010:3.0:1773941371.003958:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000aca600cf. 00000400:00000200:2.0:1773941371.004001:0:2399:0:(peer.c:3844:lnet_peer_discovered()) peer 192.168.204.114@tcp 00000400:00000200:2.0:1773941371.004005:0:2399:0:(peer.c:4118:lnet_peer_discovery()) peer 192.168.204.114@tcp(00000000614d8d0d) state 0x100111 rc 0 00000100:00000010:3.0:1773941371.004012:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000084504127. 00000400:00000200:2.0:1773941371.004013:0:2399:0:(peer.c:2421:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 192.168.204.114@tcp 00000100:00000010:3.0:1773941371.004022:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000f36993d9. 00000800:00000200:1.0:1773941371.004044:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941371.004049:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000002a8b63bd. 00000400:00000200:1.0:1773941371.004054:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941371.004059:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: ACK: OK 00000100:00000010:1.0:1773941371.004191:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000003b850681. 00000100:00000010:1.0:1773941371.004198:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000b419b047. 00000100:00000010:1.0:1773941371.004204:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000638f5872. 00000100:00000010:1.0:1773941371.004207:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000fe8a2e80. 00000100:00000010:1.0:1773941371.004213:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000ae41b37a. 00000100:00000010:1.0:1773941371.004216:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000f3d8d5bb. 00000100:00000010:1.0:1773941371.004221:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000f8afb7f5. 00000100:00000010:1.0:1773941371.004224:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000c50a25ab. 00000100:00000010:1.0:1773941371.004239:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000adb353bf. 00000100:00000010:1.0:1773941371.004243:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000004f246c26. 00000100:00000010:1.0:1773941371.004246:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000006cb71576. 00000100:00000010:1.0:1773941371.004248:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000fcbff3e3. 00000100:00000010:1.0:1773941371.004251:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000086746da8. 00000100:00000010:1.0:1773941371.004253:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000ff2e13d4. 00000100:00000010:1.0:1773941371.004257:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000002303167f. 00000100:00000010:1.0:1773941371.004259:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000018ab5740. 00000100:00000010:1.0:1773941371.004271:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000222fa6c2. 00000100:00000010:1.0:1773941371.004275:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000000802f6. 00000100:00000010:1.0:1773941371.004280:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000e4fa8f6e. 00000100:00000010:1.0:1773941371.004283:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000074d1af62. 00000100:00000010:1.0:1773941371.004288:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000fce3ceec. 00000100:00000010:1.0:1773941371.004291:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000037cce7f6. 00000100:00000010:1.0:1773941371.004296:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000d52f45bb. 00000100:00000010:1.0:1773941371.004304:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000296c314c. 00000100:00000010:1.0:1773941371.004316:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000ae0726f9. 00000100:00000010:1.0:1773941371.004319:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000032f4d0ea. 00000100:00000010:1.0:1773941371.004325:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000c442cabe. 00000100:00000010:1.0:1773941371.004327:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000082050685. 00000100:00000010:1.0:1773941371.004332:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000063639071. 00000100:00000010:1.0:1773941371.004336:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000003b023cf9. 00000100:00000010:1.0:1773941371.004342:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000481f057d. 00000100:00000010:1.0:1773941371.004345:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000004cbd41b9. 00000100:00000010:1.0:1773941371.004358:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000008d49f190. 00000100:00000010:1.0:1773941371.004362:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000c24304b7. 00000100:00000010:1.0:1773941371.004367:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000085d271d5. 00000100:00000010:1.0:1773941371.004370:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000b44ff157. 00000100:00000010:1.0:1773941371.004375:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000004db924b5. 00000100:00000010:1.0:1773941371.004378:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000e085b44e. 00000100:00000010:1.0:1773941371.004383:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000a3bc7bc8. 00000100:00000010:1.0:1773941371.004386:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000003c73a8f3. 00000100:00000010:1.0:1773941371.004396:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000075ef6dd3. 00000100:00000010:1.0:1773941371.004399:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000007002b73d. 00000100:00000010:1.0:1773941371.004402:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000b3ba29b5. 00000100:00000010:1.0:1773941371.004404:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000035866ae6. 00000100:00000010:1.0:1773941371.004407:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000009bc6be19. 00000100:00000010:1.0:1773941371.004410:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000e8307251. 00000100:00000010:1.0:1773941371.004412:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000056b85684. 00000100:00000010:1.0:1773941371.004414:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000d6144e5c. 00000100:00000010:1.0:1773941371.004424:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000009c0dd052. 00000100:00000010:1.0:1773941371.004427:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000d063fcca. 00000100:00000010:1.0:1773941371.004432:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000fff44017. 00000100:00000010:1.0:1773941371.004437:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000076aa769b. 00000100:00000010:1.0:1773941371.004443:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000009f85de99. 00000100:00000010:1.0:1773941371.004447:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000cdf1491e. 00000100:00000010:1.0:1773941371.004456:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000b0c85273. 00000100:00000010:1.0:1773941371.004459:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000c74eca4c. 00000100:00000010:1.0:1773941371.004467:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000dfc61405. 00000100:00000010:1.0:1773941371.004471:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000abefeb9d. 00000100:00000010:1.0:1773941371.004475:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000098919776. 00000100:00000010:1.0:1773941371.004478:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000032bf035f. 00000100:00000010:1.0:1773941371.004482:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000085fd35f3. 00000100:00000010:1.0:1773941371.004490:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000e3e655f5. 00000400:00000200:0.0:1773941371.004493:0:2388:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:0.1:1773941371.004497:0:2388:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 3, ksnr_connected e ksnr_max_conns 1 00000100:00000010:1.0:1773941371.004498:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000061ee2a14. 00000800:00000200:0.0:1773941371.004503:0:2388:0:(socklnd.c:1183:ksocknal_create_conn()) New conn 12345-192.168.204.114@tcp p 3.x 192.168.204.14 -> 192.168.204.114:988 incarnation:1773941325747225435 sched[0] 00000100:00000010:1.0:1773941371.004504:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000003f81775c. 00000800:00000010:0.0:1773941371.004507:0:2388:0:(socklnd.c:1191:ksocknal_create_conn()) kfreed 'hello': 144 at 00000000f1ad2f8e. 00000100:00000010:1.0:1773941371.004518:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000a3c0c14d. 00000100:00000010:1.0:1773941371.004526:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000f78d6699. 00000100:00000010:1.0:1773941371.004549:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000db9daefe. 00000100:00000010:1.0:1773941371.004553:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000154a0656. 00000100:00000010:1.0:1773941371.004558:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000002515eabc. 00000100:00000010:1.0:1773941371.004560:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000005eeca17b. 00000100:00000010:1.0:1773941371.004565:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000bd423592. 00000100:00000010:1.0:1773941371.004568:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000003facd089. 00000100:00000010:1.0:1773941371.004579:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000001391f1cc. 00000100:00000010:1.0:1773941371.004581:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000008f7fdf08. 00000100:00000010:1.0:1773941371.004584:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000decd1d8b. 00000100:00000010:1.0:1773941371.004586:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000b52d68ae. 00000100:00000010:1.0:1773941371.004589:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000001b0876e8. 00000100:00000010:1.0:1773941371.004591:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000002eb99974. 00000100:00000010:1.0:1773941371.004594:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000ddcfd021. 00000100:00000010:1.0:1773941371.004595:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000f1c2b425. 00000100:00000010:1.0:1773941371.004603:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000515c2a60. 00000100:00000010:1.0:1773941371.004605:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000f912a767. 00000100:00000010:1.0:1773941371.004609:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000005055982c. 00000100:00000010:1.0:1773941371.004611:0:6296:0:(service.c:803:ptlrpc_register_service()) kmalloced '(svcpt)': 784 at 0000000016aade8d. 00000100:00000010:1.0:1773941371.004620:0:6296:0:(service.c:627:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_array)': 12080 at 000000004ec3300f. 00000100:00000010:1.0:1773941371.004630:0:6296:0:(service.c:635:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_count)': 3020 at 000000008593e6f7. 00000100:00000010:1.0:1773941371.004633:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000b7e94785. 00000100:00000010:1.0:1773941371.004650:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000009997825. 00000100:00000010:1.0:1773941371.004652:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000cc02538f. 00000100:00000010:1.0:1773941371.004656:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000009bb1e7bf. 00000100:00000010:1.0:1773941371.004657:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000031cda21a. 00000100:00000010:1.0:1773941371.004667:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000ec42c47d. 00000100:00000010:1.0:1773941371.004669:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000cf803f82. 00000100:00000010:1.0:1773941371.004673:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000009ec5f687. 00000100:00000010:1.0:1773941371.004681:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000c4d4055c. 00000100:00000010:1.0:1773941371.004685:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000cd37f616. 00000100:00000010:1.0:1773941371.004687:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000081d63851. 00000100:00000010:1.0:1773941371.004691:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000959e87a6. 00000100:00000010:1.0:1773941371.004692:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000b7c9a043. 00000100:00000010:1.0:1773941371.004700:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000c2b7b8a1. 00000100:00000010:1.0:1773941371.004702:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000009c02709a. 00000100:00000010:1.0:1773941371.004706:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000837f3c71. 00000100:00000010:1.0:1773941371.004707:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000c4014fcd. 00000100:00000010:1.0:1773941371.004711:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000fc59f5f0. 00000100:00000010:1.0:1773941371.004713:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000291ab62d. 00000100:00000010:1.0:1773941371.004717:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000b50e1574. 00000100:00000010:1.0:1773941371.004718:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000130a8a13. 00000100:00000010:1.0:1773941371.004726:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000083ec3fd3. 00000100:00000010:1.0:1773941371.004729:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000007f7c8b4c. 00000100:00000010:1.0:1773941371.004732:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000aea2541c. 00000100:00000010:1.0:1773941371.004734:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000248f83ca. 00000100:00000010:1.0:1773941371.004738:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000006552c6da. 00000100:00000010:1.0:1773941371.004740:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000edb7e3d8. 00000100:00000010:1.0:1773941371.004744:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000003b121758. 00000100:00000010:1.0:1773941371.004745:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000b7f442c9. 00000100:00000010:1.0:1773941371.004753:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000001d09a13. 00000100:00000010:1.0:1773941371.004759:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000ab3c019e. 00000100:00000010:1.0:1773941371.004763:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000d0271a5e. 00000100:00000010:1.0:1773941371.004765:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000a449c0bc. 00000100:00000010:1.0:1773941371.004769:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000eb3d14c5. 00000100:00000010:1.0:1773941371.004770:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000c5faa66d. 00000100:00000010:1.0:1773941371.004774:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000525c02c7. 00000100:00000010:1.0:1773941371.004776:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000002d8a244. 00000100:00000010:1.0:1773941371.004784:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000d34a70a6. 00000100:00000010:1.0:1773941371.004786:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000cb9b88ae. 00000100:00000010:1.0:1773941371.004790:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000009e08db40. 00000100:00000010:1.0:1773941371.004797:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000022e3dafe. 00000100:00000010:1.0:1773941371.004801:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000a63e062e. 00000100:00000010:1.0:1773941371.004803:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000da2e1fd8. 00000100:00000010:1.0:1773941371.004807:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000003ac4b8cb. 00000100:00000010:1.0:1773941371.004809:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000ed01c986. 00000100:00000010:1.0:1773941371.004817:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000e7dc87d3. 00000100:00000010:1.0:1773941371.004819:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000006bd583fa. 00000100:00000010:1.0:1773941371.004822:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000047353a05. 00000100:00000010:1.0:1773941371.004824:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000b491f0e3. 00000100:00000010:1.0:1773941371.004828:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000003c06dc4a. 00000100:00000010:1.0:1773941371.004829:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000006966aa32. 00000100:00000010:1.0:1773941371.004833:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000069638c9e. 00000100:00000010:1.0:1773941371.004835:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000c00a3d42. 00000100:00000010:1.0:1773941371.004842:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000ce557ef2. 00000100:00000010:1.0:1773941371.004844:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000c0fe69ea. 00000100:00000010:1.0:1773941371.004847:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000045ca691a. 00000100:00000010:1.0:1773941371.004849:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000cf897be7. 00000100:00000010:1.0:1773941371.004852:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000611a6814. 00000100:00000010:1.0:1773941371.004854:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000bdc28b47. 00000100:00000010:1.0:1773941371.004856:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000cae8eaee. 00000100:00000010:1.0:1773941371.004858:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000307e2678. 00000100:00000010:1.0:1773941371.004868:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000051bb0656. 00000100:00000010:1.0:1773941371.004871:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000edccb5d9. 00000100:00000010:1.0:1773941371.004875:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000032f45b4f. 00000100:00000010:1.0:1773941371.004877:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000f53d2f86. 00000100:00000010:1.0:1773941371.004881:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000008dc6b1dd. 00000100:00000010:1.0:1773941371.004883:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000067a2b69e. 00000100:00000010:1.0:1773941371.004887:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000003bbf482f. 00000100:00000010:1.0:1773941371.004889:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000007ecc3e22. 00000100:00000010:1.0:1773941371.004896:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000bd0e276b. 00000100:00000010:1.0:1773941371.004898:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000072db9458. 00000100:00000010:1.0:1773941371.004901:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000000d14a8d0. 00000100:00000010:1.0:1773941371.004908:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000daf42088. 00000100:00000010:1.0:1773941371.004911:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000015705219. 00000100:00000010:1.0:1773941371.004912:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000203cb889. 00000100:00000010:1.0:1773941371.004915:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000ae926f50. 00000100:00000010:1.0:1773941371.004917:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000e6272dfe. 00000100:00000010:1.0:1773941371.004923:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000f6597683. 00000100:00000010:1.0:1773941371.004925:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000000c76c60b. 00000100:00000010:1.0:1773941371.004928:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000db019f19. 00000100:00000010:1.0:1773941371.004930:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000e3e942e6. 00000100:00000010:1.0:1773941371.004932:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000009f3f0cb3. 00000100:00000010:1.0:1773941371.004934:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000e0d95194. 00000100:00000010:1.0:1773941371.004937:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000004bae2448. 00000100:00000010:1.0:1773941371.004939:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000006f6a0bbe. 00000100:00000010:1.0:1773941371.004947:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000005d44ae2e. 00000100:00000010:1.0:1773941371.004950:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000006db14470. 00000100:00000010:1.0:1773941371.004953:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000049b523d9. 00000100:00000010:1.0:1773941371.004955:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000ab35bce2. 00000100:00000010:1.0:1773941371.004959:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000bbfaaee4. 00000100:00000010:1.0:1773941371.004979:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000fe1511bb. 00000100:00000010:1.0:1773941371.004985:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000008957fbd. 00000100:00000010:1.0:1773941371.004993:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000072ad4ebe. 00000100:00000010:1.0:1773941371.005003:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000e9fc0733. 00000100:00000010:1.0:1773941371.005006:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000dcf4ded7. 00000100:00000010:1.0:1773941371.005011:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000009939a9bb. 00000100:00000010:1.0:1773941371.005013:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000003cbcb35f. 00000100:00000010:1.0:1773941371.005018:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000008d7bf53. 00000100:00000010:1.0:1773941371.005020:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000002fe643f7. 00000100:00000010:1.0:1773941371.005025:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000059508cd8. 00000100:00000010:1.0:1773941371.005027:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000f5d1e128. 00000100:00000010:1.0:1773941371.005034:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000e13efe5f. 00000100:00000010:1.0:1773941371.005036:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000bceb05d3. 00000100:00000010:1.0:1773941371.005039:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000688c425f. 00000100:00000010:1.0:1773941371.005046:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000a885efe8. 00000100:00000010:1.0:1773941371.005049:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 0000000005695c60. 00000100:00000010:1.0:1773941371.005051:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000003d6bf956. 00000100:00000010:1.0:1773941371.005054:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000daeca617. 00000100:00000010:1.0:1773941371.005056:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000003676404b. 00000100:00000010:1.0:1773941371.005064:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000a63922a5. 00000100:00000010:1.0:1773941371.005066:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000fea05c7d. 00000100:00000010:1.0:1773941371.005070:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000adc1c4e7. 00000100:00000010:1.0:1773941371.005072:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000002a87f528. 00000100:00000010:1.0:1773941371.005076:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000009dd41af0. 00000100:00000010:1.0:1773941371.005077:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 00000000b65b218c. 00000100:00000010:1.0:1773941371.005081:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000009d840448. 00000100:00000010:1.0:1773941371.005082:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000041a0cd0a. 00000100:00000010:1.0:1773941371.005089:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000608ffb22. 00000100:00000010:1.0:1773941371.005091:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000080a20161. 00000100:00000010:1.0:1773941371.005094:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000c76c35f0. 00000100:00000010:1.0:1773941371.005095:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000036bc3b66. 00000100:00000010:1.0:1773941371.005098:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 000000006f3f8a86. 00000100:00000010:1.0:1773941371.005100:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000041bd6d0d. 00000100:00000010:1.0:1773941371.005106:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000b6cc4654. 00000100:00000010:1.0:1773941371.005108:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 000000008073ef8f. 00000100:00000010:1.0:1773941371.005115:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000b6e36543. 00000100:00000010:1.0:1773941371.005118:0:6296:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at 0000000017d1c4be. 00000100:00000010:1.0:1773941371.005122:0:6296:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at 00000000eea89c29. 00000400:00000200:1.0:1773941371.005124:0:6296:0:(lib-ptl.c:921:LNetSetLazyPortal()) Setting portal 17 lazy 00000020:00000010:1.0:1773941371.005174:0:6296:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 000000006e06df74. 00000020:00000010:1.0:1773941371.005178:0:6296:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 3680 at 000000003d8297df. 00000100:00000010:1.0:1773941371.005212:0:6296:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 00000000e6213a0e. 00000100:00000010:1.0:1773941371.005215:0:6296:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at 000000008c795c32. 00000100:00000010:1.0:1773941371.005218:0:6296:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 00000000f5c43d75. 00000100:00000010:1.0:1773941371.005220:0:6296:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 00000000ef93bf8f. 00000100:00000010:1.0:1773941371.005222:0:6296:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 00000000f8851250. 00000100:00000010:1.0:1773941371.005225:0:6296:0:(nrs.c:1026:nrs_svcpt_setup_locked()) kmalloced '(svcpt->scp_nrs_hp)': 112 at 000000008aebe2ce. 00000100:00000010:1.0:1773941371.005227:0:6296:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 0000000047b87108. 00000100:00000010:1.0:1773941371.005229:0:6296:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at 0000000076e676f5. 00000100:00000010:1.0:1773941371.005230:0:6296:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 00000000aa5e907d. 00000100:00000010:1.0:1773941371.005232:0:6296:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 00000000e6a34acb. 00000100:00000010:1.0:1773941371.005233:0:6296:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 00000000fe9b99be. 00000100:00000010:1.0:1773941371.005236:0:6296:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 00000000f50c4a62. 00000100:00000010:1.0:1773941371.005237:0:6296:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at 000000003fbbf815. 00000100:00000010:1.0:1773941371.005238:0:6296:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 00000000b62abe35. 00000100:00000010:1.0:1773941371.005240:0:6296:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 00000000d6082dd2. 00000100:00000010:1.0:1773941371.005241:0:6296:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 0000000071a7d5d4. 00000100:00000010:1.0:1773941371.005243:0:6296:0:(nrs.c:1026:nrs_svcpt_setup_locked()) kmalloced '(svcpt->scp_nrs_hp)': 112 at 000000004a2be8b1. 00000100:00000010:1.0:1773941371.005244:0:6296:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 00000000ebf42b9d. 00000100:00000010:1.0:1773941371.005245:0:6296:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at 0000000081db9fd0. 00000100:00000010:1.0:1773941371.005247:0:6296:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 000000000a85599e. 00000100:00000010:1.0:1773941371.005249:0:6296:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 0000000091207d54. 00000100:00000010:1.0:1773941371.005251:0:6296:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at 000000007ca890c8. 00000100:00000200:1.0:1773941371.005279:0:6296:0:(service.c:848:ptlrpc_register_service()) ldlm_canceld: Started, listening on portal 17 00000100:00000010:1.0:1773941371.005283:0:6296:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at 0000000006b5f91d. 00000100:00000010:0.0:1773941371.005498:0:6301:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at 0000000008df13a6. 00000020:00000010:0.0:1773941371.005502:0:6301:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 000000001d7d7af8. 00000020:00000010:0.0:1773941371.005506:0:6301:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000170300c3. 00000020:00000010:0.0:1773941371.005508:0:6301:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000009ef57c3c. 00000020:00000010:0.0:1773941371.005511:0:6301:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000277d230c. 00000040:00000010:0.0:1773941371.005513:0:6301:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 000000007fb04fa9. 00000020:00000010:0.0:1773941371.005516:0:6301:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000018e18d6. 00000020:00000010:0.0:1773941371.005519:0:6301:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at 000000000a64b79d. 00000020:00000010:0.0:1773941371.005522:0:6301:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 0000000089a968f7. 80000000:00000010:0.0:1773941371.005524:0:6301:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000a4b447b8. 40000000:00000010:0.0:1773941371.005526:0:6301:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 000000008f2c9e38. 00000008:00000010:0.0:1773941371.005540:0:6301:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000427c0f8a. 00020000:00000010:0.0:1773941371.005544:0:6301:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 000000002ed7380c. 00008000:00000010:0.0:1773941371.005555:0:6301:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 000000002c4ddb52. 00100000:00000010:0.0:1773941371.005559:0:6301:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 00000000617bef18. 00040000:00000010:0.0:1773941371.005562:0:6301:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 00000000760ae2b1. 00040000:00000010:0.0:1773941371.005565:0:6301:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 000000008e622f71. 00040000:00000010:0.0:1773941371.005567:0:6301:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000380f1f5b. 00000004:00000010:0.0:1773941371.005569:0:6301:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 000000000546ea8d. 00000004:00000010:0.0:1773941371.005573:0:6301:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at 000000002eedb700. 00000004:00000010:0.0:1773941371.005576:0:6301:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 00000000307730d5. 00002000:00000010:0.0:1773941371.005590:0:6301:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 00000000e80fe3a4. 00000004:00000010:0.0:1773941371.005593:0:6301:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at 00000000094191f2. 00000080:00000010:0.0:1773941371.005602:0:6301:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 00000000355189a9. 00000080:00000010:0.0:1773941371.005614:0:6301:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 00000000fdd592ec. 00000100:00000200:0.0:1773941371.005621:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005625:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d0f6384a. 00000400:00000010:0.0:1773941371.005628:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000f1ad2f8e. 00000100:00000200:0.0:1773941371.005631:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005633:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000004824a933. 00000400:00000010:0.0:1773941371.005648:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000dcadee9c. 00000100:00000200:0.0:1773941371.005651:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005652:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000077006370. 00000400:00000010:0.0:1773941371.005654:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000b0db69bd. 00000100:00000200:0.0:1773941371.005655:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005656:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f912bb24. 00000400:00000010:0.0:1773941371.005658:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000051fe9b75. 00000100:00000200:0.0:1773941371.005659:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005661:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000001b340f6c. 00000400:00000010:0.0:1773941371.005662:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000025b45560. 00000100:00000200:0.0:1773941371.005663:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005664:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000056fcfbd4. 00000400:00000010:0.0:1773941371.005665:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000dcdc80f2. 00000100:00000200:0.0:1773941371.005666:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005667:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000005ae21740. 00000400:00000010:0.0:1773941371.005669:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000a9d3c25a. 00000100:00000200:0.0:1773941371.005670:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005671:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d14480fd. 00000400:00000010:0.0:1773941371.005672:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000c8a44373. 00000100:00000200:0.0:1773941371.005673:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005674:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c93de764. 00000400:00000010:0.0:1773941371.005675:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000ab0d0f16. 00000100:00000200:0.0:1773941371.005676:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005677:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000903f0c98. 00000400:00000010:0.0:1773941371.005678:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000d53394c3. 00000100:00000200:0.0:1773941371.005679:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005686:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000001d3d4221. 00000400:00000010:0.0:1773941371.005687:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000001c19842b. 00000100:00000200:0.0:1773941371.005689:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005690:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000035b1989f. 00000400:00000010:0.0:1773941371.005692:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000071c48a42. 00000100:00000200:0.0:1773941371.005693:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005703:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000053a42571. 00000400:00000010:0.0:1773941371.005705:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000073d9a376. 00000100:00000200:0.0:1773941371.005707:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005708:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b7d915a1. 00000400:00000010:0.0:1773941371.005710:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000026d20991. 00000100:00000200:0.0:1773941371.005711:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005712:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000085f6fe0e. 00000400:00000010:0.0:1773941371.005720:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000b8092dda. 00000100:00000200:0.0:1773941371.005723:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005724:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000007f7d953d. 00000400:00000010:0.0:1773941371.005725:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000003d11e048. 00000100:00000200:0.0:1773941371.005726:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005727:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ab5c4e71. 00000400:00000010:0.0:1773941371.005728:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000007ae6c37c. 00000100:00000200:0.0:1773941371.005729:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005730:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000004fbad4f0. 00000400:00000010:0.0:1773941371.005732:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000fa82b16e. 00000100:00000200:0.0:1773941371.005733:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005734:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000008ecf84d3. 00000400:00000010:0.0:1773941371.005735:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000005619ae46. 00000100:00000200:0.0:1773941371.005736:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005737:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f8747c1c. 00000400:00000010:0.0:1773941371.005738:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000dcb438a6. 00000100:00000200:0.0:1773941371.005739:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005740:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000004c4fdb97. 00000400:00000010:0.0:1773941371.005741:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000095eeec9f. 00000100:00000200:0.0:1773941371.005742:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005743:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d40c0643. 00000400:00000010:0.0:1773941371.005744:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000d9d251bb. 00000100:00000200:0.0:1773941371.005749:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005750:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000dddba908. 00000400:00000010:0.0:1773941371.005752:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000009a201b87. 00000100:00000200:0.0:1773941371.005753:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005754:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000000620a46a. 00000400:00000010:0.0:1773941371.005756:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000509558c5. 00000100:00000200:0.0:1773941371.005757:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005758:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000afb8ec1a. 00000400:00000010:0.0:1773941371.005759:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000821190dd. 00000100:00000200:0.0:1773941371.005760:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005760:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000e8ce09dc. 00000400:00000010:0.0:1773941371.005761:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000a902c2c8. 00000100:00000200:0.0:1773941371.005763:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005763:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000365475d5. 00000400:00000010:0.0:1773941371.005764:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000b3fff780. 00000100:00000200:0.0:1773941371.005765:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005766:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000081ad72e6. 00000400:00000010:0.0:1773941371.005767:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000000254d5e5. 00000100:00000200:0.0:1773941371.005768:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005769:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000025129078. 00000400:00000010:0.0:1773941371.005770:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000183c8124. 00000100:00000200:0.0:1773941371.005771:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005772:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c1faa471. 00000400:00000010:0.0:1773941371.005773:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000b0bb6c30. 00000100:00000200:0.0:1773941371.005774:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005775:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000001a5db877. 00000400:00000010:0.0:1773941371.005776:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000013b429f4. 00000100:00000200:0.0:1773941371.005777:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005778:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b7aa1143. 00000400:00000010:0.0:1773941371.005779:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000d1df633e. 00000100:00000200:0.0:1773941371.005780:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005781:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000004458e20c. 00000400:00000010:0.0:1773941371.005782:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000004c6e88f8. 00000100:00000200:0.0:1773941371.005783:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005784:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000000f21cd89. 00000400:00000010:0.0:1773941371.005788:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000cdfe80e3. 00000100:00000200:0.0:1773941371.005790:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005791:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000063929c0. 00000400:00000010:0.0:1773941371.005792:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000010234f34. 00000100:00000200:0.0:1773941371.005793:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005794:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000092c24791. 00000400:00000010:0.0:1773941371.005801:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000dcdcbec2. 00000100:00000200:0.0:1773941371.005803:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005804:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000001b36e257. 00000400:00000010:0.0:1773941371.005806:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000004ca0cb2b. 00000100:00000200:0.0:1773941371.005807:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005808:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000006dce7e32. 00000400:00000010:0.0:1773941371.005809:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000c223d9a3. 00000100:00000200:0.0:1773941371.005810:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005811:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000802a7054. 00000400:00000010:0.0:1773941371.005812:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000e62e7d89. 00000100:00000200:0.0:1773941371.005813:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005814:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000a7f95de8. 00000400:00000010:0.0:1773941371.005815:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000054f021b4. 00000100:00000200:0.0:1773941371.005816:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005817:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000003ab17fe. 00000400:00000010:0.0:1773941371.005818:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000000c3ad64d. 00000100:00000200:0.0:1773941371.005819:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005820:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000a7e43a44. 00000400:00000010:0.0:1773941371.005822:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000015da5706. 00000100:00000200:0.0:1773941371.005823:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005824:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b5722eaf. 00000400:00000010:0.0:1773941371.005825:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000201930ac. 00000100:00000200:0.0:1773941371.005826:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005827:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ce9722d6. 00000400:00000010:0.0:1773941371.005828:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000001df5d193. 00000100:00000200:0.0:1773941371.005830:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005830:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000fb0c7654. 00000400:00000010:0.0:1773941371.005831:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000013e108e3. 00000100:00000200:0.0:1773941371.005833:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005837:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000008d8937ec. 00000400:00000010:0.0:1773941371.005838:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000008500d252. 00000100:00000200:0.0:1773941371.005840:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005841:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ff5e0555. 00000400:00000010:0.0:1773941371.005842:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000a7e16265. 00000100:00000200:0.0:1773941371.005844:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005845:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000071de027f. 00000400:00000010:0.0:1773941371.005846:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000009af9e058. 00000100:00000200:0.0:1773941371.005847:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005848:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000e91c94aa. 00000400:00000010:0.0:1773941371.005849:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000003242383d. 00000100:00000200:0.0:1773941371.005850:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005851:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000642b3ccc. 00000400:00000010:0.0:1773941371.005852:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000003b54e36b. 00000100:00000200:0.0:1773941371.005853:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005854:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000005257b937. 00000400:00000010:0.0:1773941371.005855:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000006fa353f7. 00000100:00000200:0.0:1773941371.005856:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005857:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000bdcd88ed. 00000400:00000010:0.0:1773941371.005858:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000feae3c95. 00000100:00000200:0.0:1773941371.005859:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005860:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000005d55744f. 00000400:00000010:0.0:1773941371.005861:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000004929873. 00000100:00000200:0.0:1773941371.005862:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005863:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000ccba204e. 00000400:00000010:0.0:1773941371.005864:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000b14f356d. 00000100:00000200:0.0:1773941371.005865:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005866:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c4ec7630. 00000400:00000010:0.0:1773941371.005867:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000005ca3db15. 00000100:00000200:0.0:1773941371.005868:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005869:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000007c30f93b. 00000400:00000010:0.0:1773941371.005870:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000ce4f2431. 00000100:00000200:0.0:1773941371.005871:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005872:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000016d2fc34. 00000400:00000010:0.0:1773941371.005880:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000d2a11ee6. 00000100:00000200:0.0:1773941371.005885:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005886:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c827745b. 00000400:00000010:0.0:1773941371.005888:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000009210924e. 00000100:00000200:0.0:1773941371.005890:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005899:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000039d1fe86. 00000400:00000010:0.0:1773941371.005901:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000003e2c0c55. 00000100:00000200:0.0:1773941371.005903:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005904:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000009dfc8599. 00000400:00000010:0.0:1773941371.005905:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000009abbb19c. 00000100:00000200:0.0:1773941371.005906:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005907:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000004f09a123. 00000400:00000010:0.0:1773941371.005908:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000073bcb097. 00000100:00000200:0.0:1773941371.005909:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005910:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000003b672a74. 00000400:00000010:0.0:1773941371.005911:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000393b52ad. 00000100:00000200:0.0:1773941371.005912:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005913:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000044fe3a3b. 00000400:00000010:0.0:1773941371.005914:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000e019a2cd. 00000100:00000200:0.0:1773941371.005915:0:6301:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941371.005916:0:6301:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000a55b926c. 00000400:00000010:0.0:1773941371.005917:0:6301:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000000e22fb08. 00000100:00000010:0.0:1773941371.005920:0:6301:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at 0000000006f8c6f6. 00000100:00000200:0.0:1773941371.005932:0:6301:0:(service.c:3086:ptlrpc_main()) service thread 0 (#1) started 00000100:00000010:1.0:1773941371.005953:0:6296:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at 000000003793d2da. 00000100:00000010:0.0:1773941371.006198:0:6302:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at 000000000b602239. 00000020:00000010:0.0:1773941371.006203:0:6302:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 00000000961e9174. 00000020:00000010:0.0:1773941371.006207:0:6302:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000c358c386. 00000020:00000010:0.0:1773941371.006209:0:6302:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000004b429543. 00000020:00000010:0.0:1773941371.006211:0:6302:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000b93209e6. 00000040:00000010:0.0:1773941371.006213:0:6302:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 0000000091cd0901. 00000020:00000010:0.0:1773941371.006217:0:6302:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 0000000075bd37c4. 00000020:00000010:0.0:1773941371.006220:0:6302:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at 0000000072c79cdc. 00000020:00000010:0.0:1773941371.006223:0:6302:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000b23902c2. 80000000:00000010:0.0:1773941371.006225:0:6302:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000b40893f8. 40000000:00000010:0.0:1773941371.006228:0:6302:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 0000000009cb812d. 00000008:00000010:0.0:1773941371.006232:0:6302:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000c2cecb7f. 00020000:00000010:0.0:1773941371.006243:0:6302:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 0000000002bdb9c1. 00008000:00000010:0.0:1773941371.006246:0:6302:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 00000000f90ddcaa. 00100000:00000010:0.0:1773941371.006250:0:6302:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 000000009948a441. 00040000:00000010:0.0:1773941371.006255:0:6302:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 0000000002070e6c. 00040000:00000010:0.0:1773941371.006259:0:6302:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 00000000b9fbe089. 00040000:00000010:0.0:1773941371.006261:0:6302:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000328719f4. 00000004:00000010:0.0:1773941371.006263:0:6302:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 00000000a35079fe. 00000004:00000010:0.0:1773941371.006266:0:6302:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at 000000002035716f. 00000004:00000010:0.0:1773941371.006269:0:6302:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 00000000a944e16a. 00002000:00000010:0.0:1773941371.006271:0:6302:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 000000001ab67d88. 00000004:00000010:0.0:1773941371.006273:0:6302:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at 00000000b68e9c0b. 00000080:00000010:0.0:1773941371.006275:0:6302:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 00000000582068c7. 00000080:00000010:0.0:1773941371.006277:0:6302:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 00000000c2005ebc. 00000100:00000010:0.0:1773941371.006280:0:6302:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at 00000000dd8bdafa. 00000100:00000200:0.0:1773941371.006292:0:6302:0:(service.c:3086:ptlrpc_main()) service thread 1 (#2) started 00000100:00000010:1.0:1773941371.006311:0:6296:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at 0000000037f91096. 00000100:00000010:1.0:1773941371.006480:0:6303:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at 00000000aaef6cd9. 00000020:00000010:1.0:1773941371.006482:0:6303:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 00000000be73c7d5. 00000020:00000010:1.0:1773941371.006485:0:6303:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000002f2e362c. 00000020:00000010:1.0:1773941371.006487:0:6303:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000c81fc306. 00000020:00000010:1.0:1773941371.006489:0:6303:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000888b4e38. 00000040:00000010:1.0:1773941371.006491:0:6303:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 0000000087a04e1a. 00000020:00000010:1.0:1773941371.006494:0:6303:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000f86c6762. 00000020:00000010:1.0:1773941371.006508:0:6303:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at 00000000438a62ca. 00000020:00000010:1.0:1773941371.006511:0:6303:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 0000000082453a4c. 80000000:00000010:1.0:1773941371.006513:0:6303:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000014bd048. 40000000:00000010:1.0:1773941371.006514:0:6303:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 0000000082023209. 00000008:00000010:1.0:1773941371.006518:0:6303:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000d891734b. 00020000:00000010:1.0:1773941371.006520:0:6303:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 0000000063aa129b. 00008000:00000010:1.0:1773941371.006523:0:6303:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 000000004399b14b. 00100000:00000010:1.0:1773941371.006526:0:6303:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 000000007ae4ad42. 00040000:00000010:1.0:1773941371.006527:0:6303:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 000000002a8b63bd. 00040000:00000010:1.0:1773941371.006530:0:6303:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 0000000072bbe676. 00040000:00000010:1.0:1773941371.006532:0:6303:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000cf66261e. 00000004:00000010:1.0:1773941371.006534:0:6303:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 000000009b05af63. 00000004:00000010:1.0:1773941371.006537:0:6303:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at 00000000258098eb. 00000004:00000010:1.0:1773941371.006540:0:6303:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 00000000a2cada07. 00002000:00000010:1.0:1773941371.006542:0:6303:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 00000000d9f412f6. 00000004:00000010:1.0:1773941371.006544:0:6303:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at 00000000fcdeba54. 00000080:00000010:1.0:1773941371.006546:0:6303:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 00000000ecbe5be1. 00000080:00000010:1.0:1773941371.006549:0:6303:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 0000000064d492bd. 00000100:00000010:1.0:1773941371.006551:0:6303:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at 000000001d230efe. 00000100:00000200:1.0:1773941371.006564:0:6303:0:(service.c:3086:ptlrpc_main()) service thread 2 (#3) started 00000100:00000010:0.0:1773941371.006583:0:6296:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at 000000001a1566d0. 00000100:00000010:2.0:1773941371.006768:0:6304:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at 000000006cce1d56. 00000020:00000010:2.0:1773941371.006773:0:6304:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 00000000b29584b9. 00000020:00000010:2.0:1773941371.006776:0:6304:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000961411ef. 00000020:00000010:2.0:1773941371.006779:0:6304:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 0000000055bc71fe. 00000020:00000010:2.0:1773941371.006781:0:6304:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000c79c2386. 00000040:00000010:2.0:1773941371.006789:0:6304:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 0000000075395adf. 00000020:00000010:2.0:1773941371.006791:0:6304:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000520e6dcc. 00000020:00000010:2.0:1773941371.006794:0:6304:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at 00000000db5136f9. 00000020:00000010:2.0:1773941371.006798:0:6304:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000f7a4acdd. 80000000:00000010:2.0:1773941371.006800:0:6304:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000c76fb139. 40000000:00000010:2.0:1773941371.006802:0:6304:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 00000000109b891b. 00000008:00000010:2.0:1773941371.006812:0:6304:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000eb6155fe. 00020000:00000010:2.0:1773941371.006816:0:6304:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 00000000d5595e30. 00008000:00000010:2.0:1773941371.006827:0:6304:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 00000000ebc14169. 00100000:00000010:2.0:1773941371.006830:0:6304:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 00000000ba999f65. 00040000:00000010:2.0:1773941371.006833:0:6304:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 00000000d846b213. 00040000:00000010:2.0:1773941371.006836:0:6304:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 0000000019f8caea. 00040000:00000010:2.0:1773941371.006838:0:6304:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000b402124b. 00000004:00000010:2.0:1773941371.006841:0:6304:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 000000001153a1e5. 00000004:00000010:2.0:1773941371.006844:0:6304:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at 000000008844abdf. 00000004:00000010:2.0:1773941371.006847:0:6304:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 0000000037fae717. 00002000:00000010:2.0:1773941371.006849:0:6304:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 00000000a4db5039. 00000004:00000010:2.0:1773941371.006852:0:6304:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at 00000000e0fdec3b. 00000080:00000010:2.0:1773941371.006862:0:6304:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 00000000602121dc. 00000080:00000010:2.0:1773941371.006874:0:6304:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 000000009b0a3b0b. 00000100:00000200:2.0:1773941371.006877:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.006881:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000003d6c830c. 00000400:00000010:2.0:1773941371.006884:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000057886006. 00000100:00000200:2.0:1773941371.006888:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.006889:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000009869a6f8. 00000400:00000010:2.0:1773941371.006891:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000babc027b. 00000100:00000200:2.0:1773941371.006893:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.006894:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000058e80bd4. 00000400:00000010:2.0:1773941371.006895:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000daf994b8. 00000100:00000200:2.0:1773941371.006897:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.006898:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d50a248d. 00000400:00000010:2.0:1773941371.006899:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000006c605818. 00000100:00000200:2.0:1773941371.006900:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.006901:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000007968f6b6. 00000400:00000010:2.0:1773941371.006905:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000ee7bc3e0. 00000100:00000200:2.0:1773941371.006907:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.006908:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000975f41cb. 00000400:00000010:2.0:1773941371.006910:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000562bd131. 00000100:00000200:2.0:1773941371.006911:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.006913:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000004d908706. 00000400:00000010:2.0:1773941371.006915:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000f5813a3b. 00000100:00000200:2.0:1773941371.006916:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.006917:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000005a32a9f6. 00000400:00000010:2.0:1773941371.006918:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000096293025. 00000100:00000200:2.0:1773941371.006919:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.006920:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c18e53a0. 00000400:00000010:2.0:1773941371.006921:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000004738b5e3. 00000100:00000200:2.0:1773941371.006923:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.006924:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000bf30a170. 00000400:00000010:2.0:1773941371.006925:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000080e53ddc. 00000100:00000200:2.0:1773941371.006926:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.006927:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000040f90297. 00000400:00000010:2.0:1773941371.006929:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000048fad558. 00000100:00000200:2.0:1773941371.006930:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.006931:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000f372aeb5. 00000400:00000010:2.0:1773941371.006932:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000069a156cb. 00000100:00000200:2.0:1773941371.006933:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.006934:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000d6a5618b. 00000400:00000010:2.0:1773941371.006935:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000008e8c627b. 00000100:00000200:2.0:1773941371.006936:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.006937:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000059bd9a49. 00000400:00000010:2.0:1773941371.006938:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000396c7025. 00000100:00000200:2.0:1773941371.006939:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.006940:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000003e86810. 00000400:00000010:2.0:1773941371.006941:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000d8c010f5. 00000100:00000200:2.0:1773941371.006942:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.006943:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000007ce53c1d. 00000400:00000010:2.0:1773941371.006945:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000076e42580. 00000100:00000200:2.0:1773941371.006947:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.006951:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000025f467c2. 00000400:00000010:2.0:1773941371.006953:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000087d62de6. 00000100:00000200:2.0:1773941371.006955:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007017:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b421c542. 00000400:00000010:2.0:1773941371.007024:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000f2534c13. 00000100:00000200:2.0:1773941371.007028:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007029:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000008a4d9479. 00000400:00000010:2.0:1773941371.007031:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000cb3eaef2. 00000100:00000200:2.0:1773941371.007033:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007034:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000010a72351. 00000400:00000010:2.0:1773941371.007036:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000b8aeabe9. 00000100:00000200:2.0:1773941371.007042:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007043:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000689cf30d. 00000400:00000010:2.0:1773941371.007047:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000bf7bdb80. 00000100:00000200:2.0:1773941371.007051:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007053:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000058825404. 00000400:00000010:2.0:1773941371.007054:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000095ea127f. 00000100:00000200:2.0:1773941371.007056:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007057:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000badf9bd5. 00000400:00000010:2.0:1773941371.007058:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000005a493aed. 00000100:00000200:2.0:1773941371.007059:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007060:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000007fa18d2e. 00000400:00000010:2.0:1773941371.007061:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000ddeed45d. 00000100:00000200:2.0:1773941371.007062:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007063:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000176e5aea. 00000400:00000010:2.0:1773941371.007065:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000d2b08fcd. 00000100:00000200:2.0:1773941371.007067:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007068:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000552a36d2. 00000400:00000010:2.0:1773941371.007071:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000b44d09f2. 00000100:00000200:2.0:1773941371.007072:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007074:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c7cf1935. 00000400:00000010:2.0:1773941371.007076:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000ee612fd6. 00000100:00000200:2.0:1773941371.007078:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007080:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000001a0f0a6b. 00000400:00000010:2.0:1773941371.007082:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000063d54d74. 00000100:00000200:2.0:1773941371.007091:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007094:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000cf6093e1. 00000400:00000010:2.0:1773941371.007097:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000086a42bcf. 00000100:00000200:2.0:1773941371.007099:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007101:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000056265097. 00000400:00000010:2.0:1773941371.007103:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000030f58fa5. 00000100:00000200:2.0:1773941371.007106:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007108:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000a020d187. 00000400:00000010:2.0:1773941371.007110:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000003ab0f935. 00000100:00000200:2.0:1773941371.007112:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007115:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000069f2acd. 00000400:00000010:2.0:1773941371.007117:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000ace3b56f. 00000100:00000200:2.0:1773941371.007119:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007121:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000007b3bde9. 00000400:00000010:2.0:1773941371.007123:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000f716b7d1. 00000100:00000200:2.0:1773941371.007125:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007127:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000007daf8290. 00000400:00000010:2.0:1773941371.007130:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000db89ad01. 00000100:00000200:2.0:1773941371.007132:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007134:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000003d698d4. 00000400:00000010:2.0:1773941371.007136:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000004ee6e583. 00000100:00000200:2.0:1773941371.007138:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007140:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000034392d57. 00000400:00000010:2.0:1773941371.007143:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000a1bb890b. 00000100:00000200:2.0:1773941371.007145:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007147:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000002eb70674. 00000400:00000010:2.0:1773941371.007149:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000ab6dcf29. 00000100:00000200:2.0:1773941371.007151:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007153:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000818af3bd. 00000400:00000010:2.0:1773941371.007156:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000cfe7bb7b. 00000100:00000200:2.0:1773941371.007158:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007160:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000070698e12. 00000400:00000010:2.0:1773941371.007162:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000c39660b0. 00000100:00000200:2.0:1773941371.007164:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007166:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000004389fbe4. 00000400:00000010:2.0:1773941371.007173:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000001cc328b8. 00000100:00000200:2.0:1773941371.007176:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007178:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000064da8c97. 00000400:00000010:2.0:1773941371.007180:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000001b4c8d4d. 00000100:00000200:2.0:1773941371.007182:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007184:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000004b141240. 00000400:00000010:2.0:1773941371.007187:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000001c671481. 00000100:00000200:2.0:1773941371.007189:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007190:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000bb8cc183. 00000400:00000010:2.0:1773941371.007191:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000000391b08b. 00000100:00000200:2.0:1773941371.007193:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007194:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000c6a552a3. 00000400:00000010:2.0:1773941371.007195:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000db1c8d42. 00000100:00000200:2.0:1773941371.007196:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007197:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000adb4cc86. 00000400:00000010:2.0:1773941371.007198:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000009aebee30. 00000100:00000200:2.0:1773941371.007199:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007200:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000005986d890. 00000400:00000010:2.0:1773941371.007202:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000076926838. 00000100:00000200:2.0:1773941371.007204:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007205:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000009ddb4642. 00000400:00000010:2.0:1773941371.007206:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000006f196b19. 00000100:00000200:2.0:1773941371.007207:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007208:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000005e0941d7. 00000400:00000010:2.0:1773941371.007209:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000f274ec07. 00000100:00000200:2.0:1773941371.007210:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007211:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000003c2a1ceb. 00000400:00000010:2.0:1773941371.007212:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000608d3d7b. 00000100:00000200:2.0:1773941371.007213:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007214:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000b3b1be37. 00000400:00000010:2.0:1773941371.007215:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000798564af. 00000100:00000200:2.0:1773941371.007216:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007217:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000000e8d6b17. 00000400:00000010:2.0:1773941371.007218:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000005ee23676. 00000100:00000200:2.0:1773941371.007219:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007225:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000005a1f433d. 00000400:00000010:2.0:1773941371.007227:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000009adbe6d3. 00000100:00000200:2.0:1773941371.007229:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007230:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000e0c22f44. 00000400:00000010:2.0:1773941371.007231:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000057fdbf14. 00000100:00000200:2.0:1773941371.007232:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007233:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000241b9a1d. 00000400:00000010:2.0:1773941371.007234:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000007cc1212e. 00000100:00000200:2.0:1773941371.007235:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007236:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000004dacf929. 00000400:00000010:2.0:1773941371.007237:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000a95737d1. 00000100:00000200:2.0:1773941371.007238:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007239:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000647727bf. 00000400:00000010:2.0:1773941371.007240:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000062573066. 00000100:00000200:2.0:1773941371.007241:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007242:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000002e625c4b. 00000400:00000010:2.0:1773941371.007243:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000f200c75d. 00000100:00000200:2.0:1773941371.007244:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007245:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000298bf8f1. 00000400:00000010:2.0:1773941371.007246:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000007f172998. 00000100:00000200:2.0:1773941371.007247:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007248:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000010b23169. 00000400:00000010:2.0:1773941371.007249:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000b188eb5e. 00000100:00000200:2.0:1773941371.007251:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007251:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000de11ff9c. 00000400:00000010:2.0:1773941371.007253:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 00000000c1fd9758. 00000100:00000200:2.0:1773941371.007254:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007255:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000076d7dd07. 00000400:00000010:2.0:1773941371.007256:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000006e4ce204. 00000100:00000200:2.0:1773941371.007257:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007258:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000006565b0b9. 00000400:00000010:2.0:1773941371.007259:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 000000001a955ef8. 00000100:00000200:2.0:1773941371.007260:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007260:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000006c24bf1e. 00000400:00000010:2.0:1773941371.007261:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000065998a55. 00000100:00000200:2.0:1773941371.007266:0:6304:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941371.007274:0:6304:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000a7e0501c. 00000400:00000010:2.0:1773941371.007276:0:6304:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at 0000000027286cbb. 00000100:00000010:2.0:1773941371.007279:0:6304:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at 000000001a30e2dd. 00000100:00000200:2.0:1773941371.007292:0:6304:0:(service.c:3086:ptlrpc_main()) service thread 0 (#1) started 00000100:00000010:0.0:1773941371.007313:0:6296:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at 0000000010b99a66. 00000100:00000010:2.0:1773941371.007463:0:6305:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at 0000000009f4eb57. 00000020:00000010:2.0:1773941371.007467:0:6305:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 00000000fa797970. 00000020:00000010:2.0:1773941371.007469:0:6305:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004b60ef13. 00000020:00000010:2.0:1773941371.007472:0:6305:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000a9170bad. 00000020:00000010:2.0:1773941371.007473:0:6305:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 0000000074c504cb. 00000040:00000010:2.0:1773941371.007476:0:6305:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000b52a9c35. 00000020:00000010:2.0:1773941371.007478:0:6305:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000cb6cfdbc. 00000020:00000010:2.0:1773941371.007480:0:6305:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at 0000000027b39dfa. 00000020:00000010:2.0:1773941371.007482:0:6305:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 000000009a3bb0bd. 80000000:00000010:2.0:1773941371.007484:0:6305:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000c6285e21. 40000000:00000010:2.0:1773941371.007485:0:6305:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 00000000f163671b. 00000008:00000010:2.0:1773941371.007488:0:6305:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000d83777d0. 00020000:00000010:2.0:1773941371.007489:0:6305:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 00000000ef69172c. 00008000:00000010:2.0:1773941371.007491:0:6305:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 000000009d2c153b. 00100000:00000010:2.0:1773941371.007493:0:6305:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 000000005fcbb41c. 00040000:00000010:2.0:1773941371.007495:0:6305:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 0000000039f70364. 00040000:00000010:2.0:1773941371.007497:0:6305:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 00000000818d6fbd. 00040000:00000010:2.0:1773941371.007499:0:6305:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000d3c4e6e0. 00000004:00000010:2.0:1773941371.007501:0:6305:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 00000000b75a4019. 00000004:00000010:2.0:1773941371.007503:0:6305:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at 00000000fe9612b6. 00000004:00000010:2.0:1773941371.007505:0:6305:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 0000000005ccb44c. 00002000:00000010:2.0:1773941371.007507:0:6305:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 00000000738eb728. 00000004:00000010:2.0:1773941371.007509:0:6305:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at 00000000dc81382c. 00000080:00000010:2.0:1773941371.007511:0:6305:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 0000000051bdd2a9. 00000080:00000010:2.0:1773941371.007513:0:6305:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 000000001db40be7. 00000100:00000010:2.0:1773941371.007515:0:6305:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at 0000000089069526. 00000100:00000200:2.0:1773941371.007527:0:6305:0:(service.c:3086:ptlrpc_main()) service thread 1 (#2) started 00000100:00000010:0.0:1773941371.007534:0:6296:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at 00000000ae538843. 00000100:00000010:2.0:1773941371.007697:0:6306:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at 00000000db23ff1d. 00000020:00000010:2.0:1773941371.007700:0:6306:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 000000002617c988. 00000020:00000010:2.0:1773941371.007709:0:6306:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000d9592667. 00000020:00000010:2.0:1773941371.007710:0:6306:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000005ac047f7. 00000020:00000010:2.0:1773941371.007712:0:6306:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000838ea5ba. 00000040:00000010:2.0:1773941371.007714:0:6306:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 000000000ea6e165. 00000020:00000010:2.0:1773941371.007716:0:6306:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 000000005992dc4c. 00000020:00000010:2.0:1773941371.007718:0:6306:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at 00000000d4baad73. 00000020:00000010:2.0:1773941371.007719:0:6306:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000f1c0f167. 80000000:00000010:2.0:1773941371.007721:0:6306:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 000000005cfecb70. 40000000:00000010:2.0:1773941371.007722:0:6306:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 00000000b84f0c34. 00000008:00000010:2.0:1773941371.007725:0:6306:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000a8284526. 00020000:00000010:2.0:1773941371.007726:0:6306:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 000000004786e55d. 00008000:00000010:2.0:1773941371.007728:0:6306:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 00000000ff78f94d. 00100000:00000010:2.0:1773941371.007730:0:6306:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 000000005607b013. 00040000:00000010:2.0:1773941371.007732:0:6306:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 00000000a12f8ce6. 00040000:00000010:2.0:1773941371.007741:0:6306:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 000000006d388048. 00040000:00000010:2.0:1773941371.007744:0:6306:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 000000003e96fb1f. 00000004:00000010:2.0:1773941371.007746:0:6306:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 00000000445823d3. 00000004:00000010:2.0:1773941371.007748:0:6306:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at 000000006a690269. 00000004:00000010:2.0:1773941371.007750:0:6306:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 00000000627b7143. 00002000:00000010:2.0:1773941371.007752:0:6306:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 00000000f5c32789. 00000004:00000010:2.0:1773941371.007754:0:6306:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at 000000005db08c28. 00000080:00000010:2.0:1773941371.007755:0:6306:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 0000000075ba9b4c. 00000080:00000010:2.0:1773941371.007757:0:6306:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 0000000022602145. 00000100:00000010:2.0:1773941371.007760:0:6306:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at 00000000053432d9. 00000100:00000200:2.0:1773941371.007771:0:6306:0:(service.c:3086:ptlrpc_main()) service thread 2 (#3) started 00010000:00000010:0.0:1773941371.007782:0:6296:0:(ldlm_lockd.c:3466:ldlm_setup()) kmalloced '(blp)': 168 at 00000000f433a79b. 00010000:00000010:2.0:1773941371.007906:0:6307:0:(ldlm_lockd.c:2961:ldlm_bl_thread_main()) kmalloced '(env)': 56 at 000000002b845226. 00000020:00000010:2.0:1773941371.007911:0:6307:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000ac303f04. 00000020:00000010:2.0:1773941371.007914:0:6307:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000000baf4fdd. 00000020:00000010:2.0:1773941371.007915:0:6307:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 000000007ac982b5. 00000020:00000010:2.0:1773941371.007918:0:6307:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at 0000000071f68ce7. 00000020:00000010:2.0:1773941371.007920:0:6307:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 0000000090d7ae77. 80000000:00000010:2.0:1773941371.007922:0:6307:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 0000000062107eb6. 40000000:00000010:2.0:1773941371.007923:0:6307:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 00000000ef74eab0. 00100000:00000010:2.0:1773941371.007930:0:6307:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 00000000f8620efb. 00040000:00000010:2.0:1773941371.007932:0:6307:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 00000000172f8ab3. 00040000:00000010:2.0:1773941371.007934:0:6307:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000dd6da4e8. 00002000:00000010:2.0:1773941371.007936:0:6307:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 000000005202d3f3. 00000020:00000010:2.0:1773941371.007938:0:6307:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 0000000056a1cd80. 00010000:00000010:2.0:1773941371.008091:0:6308:0:(ldlm_lockd.c:2961:ldlm_bl_thread_main()) kmalloced '(env)': 56 at 0000000091dae0f5. 00000020:00000010:2.0:1773941371.008095:0:6308:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000cf306113. 00000020:00000010:2.0:1773941371.008098:0:6308:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000007e061252. 00000020:00000010:2.0:1773941371.008101:0:6308:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 000000000230ced2. 00000020:00000010:2.0:1773941371.008104:0:6308:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at 000000003ad3d937. 00000020:00000010:2.0:1773941371.008107:0:6308:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000d134b6ec. 80000000:00000010:2.0:1773941371.008110:0:6308:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 0000000074841a1c. 40000000:00000010:2.0:1773941371.008112:0:6308:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 00000000b205456a. 00100000:00000010:2.0:1773941371.008114:0:6308:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 00000000eb005405. 00040000:00000010:2.0:1773941371.008118:0:6308:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 000000009f4ec3be. 00040000:00000010:2.0:1773941371.008121:0:6308:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000720f3354. 00002000:00000010:2.0:1773941371.008124:0:6308:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 00000000ad1fcbf5. 00000020:00000010:2.0:1773941371.008127:0:6308:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 000000006255efed. 00000020:00000010:2.0:1773941371.008261:0:6309:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000004af0c4a4. 00000020:00000010:2.0:1773941371.008264:0:6309:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000f5aafb7d. 00000040:00000010:2.0:1773941371.008266:0:6309:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000d6291442. 00000020:00000010:2.0:1773941371.008268:0:6309:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 000000000d3b06ff. 00000020:00000010:2.0:1773941371.008270:0:6309:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at 000000007cee5aa5. 00000020:00000010:2.0:1773941371.008272:0:6309:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000440609f3. 80000000:00000010:2.0:1773941371.008273:0:6309:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000a5bb948b. 40000000:00000010:2.0:1773941371.008275:0:6309:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 00000000abe574d7. 00100000:00000010:2.0:1773941371.008276:0:6309:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 00000000fd4b3bdc. 00040000:00000010:2.0:1773941371.008278:0:6309:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 000000009e3870f4. 00040000:00000010:2.0:1773941371.008280:0:6309:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 00000000e78cd98d. 00040000:00000010:2.0:1773941371.008282:0:6309:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000b22645cc. 00000004:00000010:2.0:1773941371.008284:0:6309:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 000000001d43e085. 00000004:00000010:2.0:1773941371.008286:0:6309:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at 00000000cb125d0f. 00000004:00000010:2.0:1773941371.008288:0:6309:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 0000000067a9deee. 00002000:00000010:2.0:1773941371.008301:0:6309:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 00000000fda53736. 00000004:00000010:2.0:1773941371.008304:0:6309:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at 000000008434502d. 00000020:00000010:2.0:1773941371.008306:0:6309:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 00000000e327d7da. 00000001:00000010:0.0:1773941371.008325:0:6296:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 000000005fb1038e. 00000001:00000010:0.0:1773941371.008332:0:6296:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at 00000000024d61c2. 00000020:00000010:0.0:1773941371.008351:0:6296:0:(genops.c:1283:class_new_import()) kmalloced '(imp)': 1712 at 0000000004e9e743. 00000100:00000200:0.0:1773941371.008368:0:6296:0:(events.c:560:ptlrpc_uuid_to_peer()) 192.168.204.114@tcp->12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941371.008372:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.008376:0:6296:0:(peer.c:1471:LNetPrimaryNID()) NID 192.168.204.114@tcp primary NID 192.168.204.114@tcp rc 0 00000100:00000010:0.0:1773941371.008380:0:6296:0:(connection.c:113:ptlrpc_connection_get()) kmalloced '(conn)': 40 at 00000000668f81de. 00010000:00000010:0.0:1773941371.008384:0:6296:0:(ldlm_lib.c:70:import_set_conn()) kmalloced '(imp_conn)': 88 at 00000000644d8696. 00010000:00000010:0.0:1773941371.008390:0:6296:0:(ldlm_resource.c:955:ldlm_namespace_new()) kmalloced '(ns)': 848 at 00000000d712b6a8. 00000001:00000010:0.0:1773941371.008393:0:6296:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 280 at 000000001d9c7063. 00000001:00000010:0.0:1773941371.008397:0:6296:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 16 at 0000000058eeeb79. 00000001:00000010:0.0:1773941371.008399:0:6296:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 304 at 00000000a6ff6b49. 00000001:00000010:0.0:1773941371.008402:0:6296:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 304 at 00000000cd4cb595. 00000020:00000010:0.0:1773941371.008459:0:6296:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000fbeb2fe8. 00000020:00000010:0.0:1773941371.008464:0:6296:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 96 at 00000000a6264313. 00000020:00000010:0.0:1773941371.008506:0:6296:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000b2e55b59. 00000020:00000010:0.0:1773941371.008509:0:6296:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 352 at 000000007f07c3e0. 00000040:00000010:0.0:1773941371.008521:0:6296:0:(llog_obd.c:26:llog_new_ctxt()) kmalloced '(ctxt)': 168 at 000000003f844e03. 00000020:00000010:1.0:1773941371.008845:0:6296:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 00000000b2ef6fef. 00000040:00000010:1.0:1773941371.008854:0:6296:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at 00000000fb0d7ffe. 00000020:00000010:1.0:1773941371.008858:0:6296:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at 0000000003d81041. 00000020:00000010:1.0:1773941371.008864:0:6296:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at 000000006b47a18f. 00040000:00000010:1.0:1773941371.008871:0:6296:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at 0000000072edce6b. 00040000:00000010:1.0:1773941371.008876:0:6296:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at 00000000c7ea8da2. 00000020:00000010:1.0:1773941371.008880:0:6296:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000b6d0fb77. 00000020:00000010:1.0:1773941371.008885:0:6296:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 00000000cbfe9690. 00000020:00000010:1.0:1773941371.008890:0:6296:0:(tgt_main.c:768:tgt_ses_key_fini()) slab-freed '(session)': 200 at 00000000211306bd. 00000008:00000010:1.0:1773941371.008893:0:6296:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at 0000000020073e0c. 00020000:00000010:1.0:1773941371.008897:0:6296:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at 00000000e37deccf. 00008000:00000010:1.0:1773941371.008902:0:6296:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at 000000007cfd2609. 00000080:00000010:1.0:1773941371.008907:0:6296:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at 0000000070a0e965. 00000020:00000010:1.0:1773941371.008910:0:6296:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000004ff72262. 00000020:00000010:1.0:1773941371.008915:0:6296:0:(obd_mount.c:168:do_lcfg()) kfreed 'lcfg': 104 at 00000000f64f1c48. 00000020:00000010:1.0:1773941371.008930:0:6296:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at 00000000d81868e0. 00000400:00000200:1.0:1773941371.008946:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.008951:0:6296:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000010:1.0:1773941371.009022:0:6296:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000001eb38943. 02000000:00000010:1.0:1773941371.009037:0:6296:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at 00000000950bf78e. 00000020:00000010:1.0:1773941371.009082:0:6296:0:(obd_mount.c:561:lustre_start_mgc()) kfreed 'uuid': 40 at 00000000acd85031. 00000020:00000010:1.0:1773941371.009086:0:6296:0:(obd_mount.c:562:lustre_start_mgc()) kfreed 'data': 192 at 00000000cd8776d0. 00000020:00000010:1.0:1773941371.009089:0:6296:0:(obd_mount.c:563:lustre_start_mgc()) kfreed 'mgcname': 23 at 000000007af8ad59. 00000080:00000010:1.0:1773941371.009095:0:6296:0:(llite_lib.c:1399:ll_fill_super()) kmalloced '(cfg)': 88 at 00000000794ef9e2. 00000080:00000010:1.0:1773941371.009099:0:6296:0:(llite_lib.c:95:ll_init_sbi()) kmalloced '(sbi)': 1376 at 000000004ad80115. 00000020:00000010:3.0:1773941371.009116:0:2400:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at 0000000035c3fc74. 00000020:00000010:3.0:1773941371.009124:0:2400:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 000000001b8a99ba. 80000000:00000010:3.0:1773941371.009127:0:2400:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 000000003033795b. 40000000:00000010:3.0:1773941371.009137:0:2400:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 00000000b9445e27. 00000008:00000010:3.0:1773941371.009148:0:2400:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000291433d9. 00020000:00000010:3.0:1773941371.009152:0:2400:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 000000006d0ac904. 00008000:00000010:3.0:1773941371.009162:0:2400:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 000000000e14e548. 00100000:00000010:3.0:1773941371.009166:0:2400:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 000000000ea19666. 00040000:00000010:3.0:1773941371.009170:0:2400:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 0000000091445819. 00040000:00000010:3.0:1773941371.009173:0:2400:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 0000000084ae58ed. 00040000:00000010:3.0:1773941371.009176:0:2400:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000cc26eaf6. 00000004:00000010:3.0:1773941371.009179:0:2400:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 00000000064068f7. 00000004:00000010:3.0:1773941371.009185:0:2400:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at 00000000ca4edc0e. 00000020:00000010:1.0:1773941371.009185:0:6296:0:(cl_page.c:1380:cl_cache_init()) kmalloced '(cache)': 152 at 000000000f78e135. 00000004:00000010:3.0:1773941371.009189:0:2400:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 000000004805fe93. 00000080:00000010:1.0:1773941371.009191:0:6296:0:(llite_lib.c:129:ll_init_sbi()) kmalloced '(sbi->ll_foreign_symlink_prefix)': 6 at 00000000349a9832. 00002000:00000010:3.0:1773941371.009197:0:2400:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 000000002cae11b9. 00000080:00000010:1.0:1773941371.009198:0:6296:0:(llite_lib.c:136:ll_init_sbi()) kmalloced '(sbi->ll_foreign_symlink_upcall)': 5 at 0000000047a8d675. 00000004:00000010:3.0:1773941371.009204:0:2400:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at 0000000029b16e30. 00000080:00000010:3.0:1773941371.009219:0:2400:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 00000000c21edecb. 00000080:00000010:3.0:1773941371.009231:0:2400:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 0000000078873819. 00000008:00000010:3.0:1773941371.009245:0:2400:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at 000000003e4042bc. 00020000:00000010:3.0:1773941371.009257:0:2400:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at 00000000414a7abe. 00008000:00000010:3.0:1773941371.009273:0:2400:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at 00000000cf36e86e. 00000080:00000010:3.0:1773941371.009287:0:2400:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at 000000005daaf60e. 02000000:00000010:3.0:1773941371.009309:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000aa518d29. 00000400:00000200:3.0:1773941371.009317:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941371.009322:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000009ef7eb24. 00000400:00000010:3.0:1773941371.009338:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000039f9e5d. 00000100:00000200:3.0:1773941371.009345:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291463296, portal 25 00000400:00000010:3.0:1773941371.009352:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000006ca26ba7. 00000100:00000200:3.0:1773941371.009355:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291463296, offset 0 00000400:00000200:3.0:1773941371.009377:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941371.009386:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941371.009394:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941371.009398:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941371.009402:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941371.009407:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941371.009410:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941371.009425:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [4:4:256:4294967295] 192.168.204.114@tcp LPNI seq info [4:4:8:4294967295] 00000400:00000200:3.0:1773941371.009432:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941371.009444:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:3.0:1773941371.009448:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:3.0:1773941371.009454:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000000ab1681f. 00000800:00000200:3.0:1773941371.009460:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006663d74c] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:3.0:1773941371.009465:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:3.0:1773941371.009468:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 000000000ab1681f type 1, nob 616 niov 1 nkiov 1 00000400:00000200:3.0:1773941371.009485:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000010:3.0:1773941371.009491:0:2400:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000009574c029. 00000100:00000010:3.0:1773941371.009508:0:2400:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 000000001d24849f. 00000800:00000200:0.0:1773941371.009569:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941371.009572:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000ab1681f. 00000400:00000200:0.0:1773941371.009576:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941371.009582:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941371.009585:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000006ca26ba7 00000400:00000010:0.0:1773941371.009587:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 000000006ca26ba7. 00000100:00000200:0.0:1773941371.009596:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000001eb38943 x1860112291463296/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941381 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1773941371.010037:0:6296:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000b4468275. 00000020:00000010:1.0:1773941371.010041:0:6296:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 1440 at 0000000003d81041. 00000020:00000010:1.0:1773941371.010063:0:6296:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 0000000097293963. 00000020:00000010:1.0:1773941371.010067:0:6296:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 576 at 00000000c01f571a. 00000020:00000010:1.0:1773941371.010088:0:6296:0:(obd_mount.c:67:lustre_process_log()) kmalloced '(bufs)': 104 at 00000000008ee5fb. 00000020:00000010:1.0:1773941371.010093:0:6296:0:(obd_mount.c:76:lustre_process_log()) kmalloced '(lcfg)': 184 at 00000000cd8776d0. 00000020:00000010:1.0:1773941371.010097:0:6296:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000003960c481. 00000020:00000010:1.0:1773941371.010101:0:6296:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000004ff72262. 00000040:00000010:1.0:1773941371.010109:0:6296:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000b6d0fb77. 00000020:00000010:1.0:1773941371.010114:0:6296:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000f31de01f. 00000020:00000010:1.0:1773941371.010117:0:6296:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000c7ea8da2. 00040000:00000010:1.0:1773941371.010122:0:6296:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 0000000025979b8f. 00040000:00000010:1.0:1773941371.010126:0:6296:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 000000006b47a18f. 10000000:00000010:1.0:1773941371.010135:0:6296:0:(mgc_request.c:197:do_config_log_add()) kmalloced '(cld)': 287 at 00000000fb0d7ffe. 02000000:00000010:1.0:1773941371.010141:0:6296:0:(sec_config.c:546:sptlrpc_conf_get()) kmalloced '(conf)': 120 at 00000000846ae52a. 00000100:00000010:1.0:1773941371.010147:0:6296:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000022b3efad. 02000000:00000010:1.0:1773941371.010153:0:6296:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 00000000b2ef6fef. 00010000:00000010:1.0:1773941371.010164:0:6296:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at 00000000383cee2e. 00010000:00000010:1.0:1773941371.010171:0:6296:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at 00000000007734b2. 00000020:00000010:1.0:1773941371.010175:0:6296:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 128 at 000000000c96dea2. 00000100:00000010:1.0:1773941371.010193:0:6296:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 0000000079440604. 00000400:00000200:0.0:1773941371.010493:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.010498:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941371.010503:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.010506:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.010511:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.010514:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324600080 00000400:00000200:0.0:1773941371.010517:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.010521:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0x221 [1] + 0 00000400:00000200:0.0:1773941371.010535:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941371.010538:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:0.0:1773941371.010544:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000001eb38943 x1860112291463296/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941381 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1773941371.010591:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 000000009ef7eb24. 00000400:00000200:3.0:1773941371.010598:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000039f9e5d 00000400:00000010:3.0:1773941371.010600:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000039f9e5d. 00000100:00000200:3.0:1773941371.010609:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000001eb38943 x1860112291463296/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941381 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941371.010618:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000001eb38943 x1860112291463296/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941381 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1773941371.010717:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000aa518d29. 02000000:00000010:3.0:1773941371.010724:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at 00000000950bf78e. 00000100:00000010:3.0:1773941371.010731:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000001eb38943. 02000000:00000010:1.0:1773941371.010736:0:6296:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000042f5a636. 00000400:00000200:1.0:1773941371.010742:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941371.010746:0:6296:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000070b48f2b. 00000400:00000010:1.0:1773941371.010761:0:6296:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000cee77f6a. 00000100:00000200:1.0:1773941371.010765:0:6296:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291463424, portal 25 00000400:00000010:1.0:1773941371.010769:0:6296:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000c15d06a0. 00000100:00000200:1.0:1773941371.010771:0:6296:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291463424, offset 0 00000400:00000200:1.0:1773941371.010776:0:6296:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941371.010781:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.010786:0:6296:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941371.010789:0:6296:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941371.010793:0:6296:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941371.010796:0:6296:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941371.010799:0:6296:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:1.0:1773941371.010802:0:6296:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [5:5:256:4294967295] 192.168.204.114@tcp LPNI seq info [5:5:8:4294967295] 00000400:00000200:1.0:1773941371.010805:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.010812:0:6296:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:1.0:1773941371.010815:0:6296:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:1.0:1773941371.010823:0:6296:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000d7144bca. 00000800:00000200:1.0:1773941371.010827:0:6296:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006663d74c] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:1.0:1773941371.010832:0:6296:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:1.0:1773941371.010835:0:6296:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 00000000d7144bca type 1, nob 424 niov 1 nkiov 1 00000400:00000200:1.0:1773941371.010849:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:0.0:1773941371.010898:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941371.010902:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000d7144bca. 00000400:00000200:0.0:1773941371.010905:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941371.010909:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941371.010913:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000c15d06a0 00000400:00000010:0.0:1773941371.010914:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000c15d06a0. 00000100:00000200:0.0:1773941371.010919:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000022b3efad x1860112291463424/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941387 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1773941371.011603:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.011608:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941371.011617:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.011621:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.011625:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.011627:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 344 into portal 25 MB=0x69bc324600100 00000400:00000200:0.0:1773941371.011645:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.011650:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 344/344 into md 0x229 [1] + 224 00000400:00000200:0.0:1773941371.011661:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941371.011664:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:0.0:1773941371.011669:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000022b3efad x1860112291463424/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941387 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1773941371.011698:0:6296:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 0000000070b48f2b. 00000400:00000200:1.0:1773941371.011705:0:6296:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000cee77f6a 00000400:00000010:1.0:1773941371.011707:0:6296:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000cee77f6a. 00000100:00000200:1.0:1773941371.011714:0:6296:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000022b3efad x1860112291463424/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941387 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941371.011726:0:6296:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000022b3efad x1860112291463424/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941387 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1773941371.011748:0:6296:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at 0000000079440604. 02000000:00000010:1.0:1773941371.011771:0:6296:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000042f5a636. 02000000:00000010:1.0:1773941371.011776:0:6296:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 00000000b2ef6fef. 00000100:00000010:1.0:1773941371.011781:0:6296:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000022b3efad. 10000000:00000010:1.0:1773941371.011791:0:6296:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at 000000008922d770. 00000020:00000010:1.0:1773941371.011796:0:6296:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000a7f730fa. 00000020:00000010:1.0:1773941371.011802:0:6296:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000ebbc95b6. 00000040:00000010:1.0:1773941371.011807:0:6296:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000905556b5. 00000020:00000010:1.0:1773941371.011813:0:6296:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000fc723563. 00000020:00000010:1.0:1773941371.011818:0:6296:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000e0bb6f67. 80000000:00000010:1.0:1773941371.011822:0:6296:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000862b1b3f. 00040000:00000010:1.0:1773941371.011827:0:6296:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 000000001dd188c4. 20000000:00000010:1.0:1773941371.011832:0:6296:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at 0000000045d8fe82. 00000040:00000010:1.0:1773941371.011841:0:6296:0:(llog.c:42:llog_alloc_handle()) kmalloced '(loghandle)': 352 at 00000000aefdd398. 00000100:00000010:1.0:1773941371.011847:0:6296:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000022b3efad. 02000000:00000010:1.0:1773941371.011857:0:6296:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 00000000479775a9. 00000100:00000010:1.0:1773941371.011868:0:6296:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 0000000079440604. 02000000:00000010:1.0:1773941371.011880:0:6296:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 000000008b6f0a19. 00000400:00000200:1.0:1773941371.011886:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941371.011898:0:6296:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000070b48f2b. 00000400:00000010:1.0:1773941371.011903:0:6296:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000cee77f6a. 00000100:00000200:1.0:1773941371.011908:0:6296:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112291463552, portal 25 00000400:00000010:1.0:1773941371.011912:0:6296:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000551f50cf. 00000100:00000200:1.0:1773941371.011914:0:6296:0:(niobuf.c:94:ptl_send_buf()) Sending 512 bytes to portal 26, xid 1860112291463552, offset 0 00000400:00000200:1.0:1773941371.011922:0:6296:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941371.011928:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.011935:0:6296:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941371.011939:0:6296:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941371.011943:0:6296:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941371.011947:0:6296:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941371.011950:0:6296:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:1.0:1773941371.011954:0:6296:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [6:6:256:4294967295] 192.168.204.114@tcp LPNI seq info [6:6:8:4294967295] 00000400:00000200:1.0:1773941371.011985:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.011997:0:6296:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:1.0:1773941371.012005:0:6296:0:(socklnd_cb.c:1001:ksocknal_send()) sending 512 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:1.0:1773941371.012010:0:6296:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000002dfb5e5b. 00000800:00000200:1.0:1773941371.012015:0:6296:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006663d74c] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:1.0:1773941371.012022:0:6296:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:1.0:1773941371.012026:0:6296:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 000000002dfb5e5b type 1, nob 608 niov 1 nkiov 1 00000400:00000200:1.0:1773941371.012040:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:0.0:1773941371.012110:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941371.012116:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000002dfb5e5b. 00000400:00000200:0.0:1773941371.012122:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941371.012128:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941371.012133:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000551f50cf 00000400:00000010:0.0:1773941371.012136:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000551f50cf. 00000100:00000200:0.0:1773941371.012145:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000022b3efad x1860112291463552/t0(0) o501->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 512/272 e 0 to 0 dl 1773941387 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1773941371.013013:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.013018:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941371.013024:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.013029:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.013033:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.013036:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 272 into portal 25 MB=0x69bc324600180 00000400:00000200:0.0:1773941371.013039:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.013041:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 272/272 into md 0x239 [1] + 224 00000400:00000200:0.0:1773941371.013055:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941371.013067:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:0.0:1773941371.013079:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000022b3efad x1860112291463552/t0(0) o501->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 512/272 e 0 to 0 dl 1773941387 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1773941371.013121:0:6296:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 0000000070b48f2b. 00000400:00000200:1.0:1773941371.013127:0:6296:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000cee77f6a 00000400:00000010:1.0:1773941371.013130:0:6296:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000cee77f6a. 00000100:00000200:1.0:1773941371.013139:0:6296:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000022b3efad x1860112291463552/t0(0) o501->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 512/272 e 0 to 0 dl 1773941387 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941371.013149:0:6296:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000022b3efad x1860112291463552/t0(0) o501->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 512/272 e 0 to 0 dl 1773941387 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1773941371.013166:0:6296:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at 0000000079440604. 02000000:00000010:1.0:1773941371.013172:0:6296:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 000000008b6f0a19. 02000000:00000010:1.0:1773941371.013176:0:6296:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 00000000479775a9. 00000100:00000010:1.0:1773941371.013188:0:6296:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000022b3efad. 00000040:00000010:1.0:1773941371.013196:0:6296:0:(llog.c:72:llog_free_handle()) kfreed 'loghandle': 352 at 00000000aefdd398. 00000020:00000010:1.0:1773941371.013202:0:6296:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 00000000ebbc95b6. 00000040:00000010:1.0:1773941371.013206:0:6296:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at 00000000905556b5. 00000020:00000010:1.0:1773941371.013210:0:6296:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at 00000000fc723563. 00000020:00000010:1.0:1773941371.013215:0:6296:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at 00000000e0bb6f67. 80000000:00000010:1.0:1773941371.013220:0:6296:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at 00000000862b1b3f. 00040000:00000010:1.0:1773941371.013224:0:6296:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at 000000001dd188c4. 20000000:00000010:1.0:1773941371.013228:0:6296:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at 0000000045d8fe82. 00000020:00000010:1.0:1773941371.013230:0:6296:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000a7f730fa. 10000000:00000010:1.0:1773941371.013234:0:6296:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at 000000008922d770. 10000000:00000010:1.0:1773941371.013261:0:6296:0:(mgc_request.c:197:do_config_log_add()) kmalloced '(cld)': 279 at 00000000a7f730fa. 10000000:00000010:1.0:1773941371.013269:0:6296:0:(mgc_request.c:197:do_config_log_add()) kmalloced '(cld)': 286 at 0000000045d8fe82. 10000000:00000010:1.0:1773941371.013278:0:6296:0:(mgc_request.c:197:do_config_log_add()) kmalloced '(cld)': 285 at 000000001dd188c4. 00000100:00000010:1.0:1773941371.013287:0:6296:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000022b3efad. 02000000:00000010:1.0:1773941371.013297:0:6296:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 00000000e0bb6f67. 00010000:00000010:1.0:1773941371.013314:0:6296:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at 00000000626e616c. 00000100:00000010:1.0:1773941371.013329:0:6296:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 0000000079440604. 02000000:00000010:1.0:1773941371.013340:0:6296:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000042f5a636. 00000400:00000200:1.0:1773941371.013346:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941371.013351:0:6296:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000070b48f2b. 00000400:00000010:1.0:1773941371.013357:0:6296:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000cee77f6a. 00000100:00000200:1.0:1773941371.013361:0:6296:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291463680, portal 25 00000400:00000010:1.0:1773941371.013366:0:6296:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000c2c37ee1. 00000100:00000200:1.0:1773941371.013369:0:6296:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291463680, offset 0 00000400:00000200:1.0:1773941371.013377:0:6296:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941371.013383:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.013387:0:6296:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941371.013390:0:6296:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941371.013394:0:6296:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941371.013397:0:6296:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941371.013399:0:6296:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:1.0:1773941371.013403:0:6296:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [7:7:256:4294967295] 192.168.204.114@tcp LPNI seq info [7:7:8:4294967295] 00000400:00000200:1.0:1773941371.013418:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.013427:0:6296:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:1.0:1773941371.013435:0:6296:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:1.0:1773941371.013439:0:6296:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000064603a4c. 00000800:00000200:1.0:1773941371.013448:0:6296:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006663d74c] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:1.0:1773941371.013454:0:6296:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:1.0:1773941371.013457:0:6296:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 0000000064603a4c type 1, nob 424 niov 1 nkiov 1 00000400:00000200:1.0:1773941371.013471:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:0.0:1773941371.013531:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941371.013534:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000064603a4c. 00000400:00000200:0.0:1773941371.013537:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941371.013541:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941371.013543:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000c2c37ee1 00000400:00000010:0.0:1773941371.013545:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000c2c37ee1. 00000100:00000200:0.0:1773941371.013550:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000022b3efad x1860112291463680/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941387 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1773941371.014138:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.014146:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941371.014154:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.014159:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.014164:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.014171:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 344 into portal 25 MB=0x69bc324600200 00000400:00000200:0.0:1773941371.014175:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.014181:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 344/344 into md 0x249 [1] + 224 00000400:00000200:0.0:1773941371.014194:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941371.014201:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:0.0:1773941371.014212:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000022b3efad x1860112291463680/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941387 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1773941371.014261:0:6296:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 0000000070b48f2b. 00000400:00000200:1.0:1773941371.014270:0:6296:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000cee77f6a 00000400:00000010:1.0:1773941371.014272:0:6296:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000cee77f6a. 00000100:00000200:1.0:1773941371.014281:0:6296:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000022b3efad x1860112291463680/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941387 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941371.014293:0:6296:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000022b3efad x1860112291463680/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941387 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1773941371.014325:0:6296:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at 0000000079440604. 02000000:00000010:1.0:1773941371.014341:0:6296:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000042f5a636. 02000000:00000010:1.0:1773941371.014345:0:6296:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 00000000e0bb6f67. 00000100:00000010:1.0:1773941371.014348:0:6296:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000022b3efad. 10000000:00000010:1.0:1773941371.014357:0:6296:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at 000000008922d770. 00000020:00000010:1.0:1773941371.014361:0:6296:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000e0bb6f67. 00000020:00000010:1.0:1773941371.014364:0:6296:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000905556b5. 00000040:00000010:1.0:1773941371.014367:0:6296:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000ebbc95b6. 00000020:00000010:1.0:1773941371.014371:0:6296:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000fc723563. 00000020:00000010:1.0:1773941371.014375:0:6296:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000aefdd398. 80000000:00000010:1.0:1773941371.014378:0:6296:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000862b1b3f. 00040000:00000010:1.0:1773941371.014382:0:6296:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000479775a9. 20000000:00000010:1.0:1773941371.014386:0:6296:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at 000000008b6f0a19. 00000040:00000010:1.0:1773941371.014393:0:6296:0:(llog.c:42:llog_alloc_handle()) kmalloced '(loghandle)': 352 at 00000000df04275e. 00000100:00000010:1.0:1773941371.014402:0:6296:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000022b3efad. 02000000:00000010:1.0:1773941371.014422:0:6296:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 000000004620942e. 00000100:00000010:1.0:1773941371.014431:0:6296:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 0000000079440604. 02000000:00000010:1.0:1773941371.014441:0:6296:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000f6f597b9. 00000400:00000200:1.0:1773941371.014447:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941371.014451:0:6296:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000070b48f2b. 00000400:00000010:1.0:1773941371.014456:0:6296:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000cee77f6a. 00000100:00000200:1.0:1773941371.014460:0:6296:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112291463808, portal 25 00000400:00000010:1.0:1773941371.014466:0:6296:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b0aed1e9. 00000100:00000200:1.0:1773941371.014468:0:6296:0:(niobuf.c:94:ptl_send_buf()) Sending 512 bytes to portal 26, xid 1860112291463808, offset 0 00000400:00000200:1.0:1773941371.014475:0:6296:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941371.014481:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.014488:0:6296:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941371.014492:0:6296:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941371.014495:0:6296:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941371.014498:0:6296:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941371.014501:0:6296:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:1.0:1773941371.014505:0:6296:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [8:8:256:4294967295] 192.168.204.114@tcp LPNI seq info [8:8:8:4294967295] 00000400:00000200:1.0:1773941371.014509:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.014520:0:6296:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:1.0:1773941371.014527:0:6296:0:(socklnd_cb.c:1001:ksocknal_send()) sending 512 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:1.0:1773941371.014531:0:6296:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000adbd3943. 00000800:00000200:1.0:1773941371.014536:0:6296:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006663d74c] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:1.0:1773941371.014542:0:6296:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:1.0:1773941371.014545:0:6296:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 00000000adbd3943 type 1, nob 608 niov 1 nkiov 1 00000400:00000200:1.0:1773941371.014562:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:1.0:1773941371.014652:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941371.014656:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000adbd3943. 00000400:00000200:1.0:1773941371.014660:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941371.014665:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941371.014669:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b0aed1e9 00000400:00000010:1.0:1773941371.014671:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000b0aed1e9. 00000100:00000200:1.0:1773941371.014676:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000022b3efad x1860112291463808/t0(0) o501->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 512/272 e 0 to 0 dl 1773941387 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1773941371.015248:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.015253:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941371.015260:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.015264:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.015269:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.015273:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 272 into portal 25 MB=0x69bc324600280 00000400:00000200:1.0:1773941371.015277:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.015284:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 272/272 into md 0x259 [1] + 224 00000400:00000200:1.0:1773941371.015296:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941371.015303:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:1.0:1773941371.015309:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000022b3efad x1860112291463808/t0(0) o501->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 512/272 e 0 to 0 dl 1773941387 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1773941371.015365:0:6296:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 0000000070b48f2b. 00000400:00000200:2.0:1773941371.015374:0:6296:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000cee77f6a 00000400:00000010:2.0:1773941371.015377:0:6296:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000cee77f6a. 00000100:00000200:2.0:1773941371.015386:0:6296:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000022b3efad x1860112291463808/t0(0) o501->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 512/272 e 0 to 0 dl 1773941387 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941371.015398:0:6296:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000022b3efad x1860112291463808/t0(0) o501->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 512/272 e 0 to 0 dl 1773941387 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1773941371.015417:0:6296:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at 0000000079440604. 02000000:00000010:2.0:1773941371.015426:0:6296:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000f6f597b9. 02000000:00000010:2.0:1773941371.015430:0:6296:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 000000004620942e. 00000100:00000010:2.0:1773941371.015432:0:6296:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000022b3efad. 00000040:00000010:2.0:1773941371.015453:0:6296:0:(llog.c:412:llog_init_handle()) kmalloced '(llh)': 8192 at 000000009ed390fc. 00000100:00000010:2.0:1773941371.015471:0:6296:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005b838395. 02000000:00000010:2.0:1773941371.015482:0:6296:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 0000000014a40cca. 00000100:00000010:2.0:1773941371.015492:0:6296:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 000000003f70a46a. 02000000:00000010:2.0:1773941371.015516:0:6296:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 16384 at 000000008c982916. 00000400:00000200:2.0:1773941371.015523:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941371.015528:0:6296:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000000bbef44f. 00000400:00000010:2.0:1773941371.015532:0:6296:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 184 at 0000000058cd8f53. 00000100:00000200:2.0:1773941371.015536:0:6296:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 16384 bytes, xid 1860112291463936, portal 25 00000400:00000010:2.0:1773941371.015542:0:6296:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b09eb5a8. 00000100:00000200:2.0:1773941371.015545:0:6296:0:(niobuf.c:94:ptl_send_buf()) Sending 272 bytes to portal 26, xid 1860112291463936, offset 0 00000400:00000200:2.0:1773941371.015553:0:6296:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941371.015559:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941371.015569:0:6296:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941371.015573:0:6296:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941371.015577:0:6296:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941371.015580:0:6296:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941371.015583:0:6296:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:2.0:1773941371.015595:0:6296:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [9:9:256:4294967295] 192.168.204.114@tcp LPNI seq info [9:9:8:4294967295] 00000400:00000200:2.0:1773941371.015601:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941371.015614:0:6296:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:2.0:1773941371.015624:0:6296:0:(socklnd_cb.c:1001:ksocknal_send()) sending 272 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:2.0:1773941371.015628:0:6296:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000729f4b1f. 00000800:00000200:2.0:1773941371.015647:0:6296:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006663d74c] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:2.0:1773941371.015653:0:6296:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:2.0:1773941371.015657:0:6296:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 00000000729f4b1f type 1, nob 368 niov 1 nkiov 1 00000400:00000200:2.0:1773941371.015680:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:1.0:1773941371.015739:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941371.015744:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000729f4b1f. 00000400:00000200:1.0:1773941371.015750:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941371.015755:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941371.015760:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b09eb5a8 00000400:00000010:1.0:1773941371.015762:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000b09eb5a8. 00000100:00000200:1.0:1773941371.015769:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000005b838395 x1860112291463936/t0(0) o503->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8416 e 0 to 0 dl 1773941387 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1773941371.016716:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.016722:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941371.016730:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.016735:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.016739:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.016742:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 8416 into portal 25 MB=0x69bc324600300 00000400:00000200:1.0:1773941371.016746:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.016752:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 8416/8416 into md 0x269 [4] + 224 00000400:00000200:1.0:1773941371.016792:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941371.016804:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:1.0:1773941371.016810:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000005b838395 x1860112291463936/t0(0) o503->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8416 e 0 to 0 dl 1773941387 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1773941371.016862:0:6296:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 000000000bbef44f. 00000400:00000200:2.0:1773941371.016869:0:6296:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000058cd8f53 00000400:00000010:2.0:1773941371.016871:0:6296:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 184 at 0000000058cd8f53. 00000100:00000200:2.0:1773941371.016878:0:6296:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000005b838395 x1860112291463936/t0(0) o503->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8416 e 0 to 0 dl 1773941387 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941371.016889:0:6296:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000005b838395 x1860112291463936/t0(0) o503->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8416 e 0 to 0 dl 1773941387 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1773941371.016906:0:6296:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at 000000003f70a46a. 02000000:00000010:2.0:1773941371.016913:0:6296:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 16384 at 000000008c982916. 02000000:00000010:2.0:1773941371.016924:0:6296:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 0000000014a40cca. 00000100:00000010:2.0:1773941371.016927:0:6296:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000005b838395. 00000040:00000010:2.0:1773941371.016936:0:6296:0:(llog.c:945:llog_process_or_fork()) kmalloced '(lpi)': 96 at 00000000984f4721. 00000020:00000010:0.0:1773941371.018029:0:6315:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000d6559bf6. 00000020:00000010:0.0:1773941371.018036:0:6315:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000740c71ac. 00000040:00000010:0.0:1773941371.018039:0:6315:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 0000000090049ec9. 00000020:00000010:0.0:1773941371.018055:0:6315:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 0000000046233ca4. 00000020:00000010:0.0:1773941371.018061:0:6315:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at 00000000fc35371f. 00000020:00000010:0.0:1773941371.018064:0:6315:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000661742f1. 80000000:00000010:0.0:1773941371.018067:0:6315:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000c21f02c8. 40000000:00000010:0.0:1773941371.018075:0:6315:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 00000000d22667de. 00100000:00000010:0.0:1773941371.018080:0:6315:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 000000000a654f30. 00040000:00000010:0.0:1773941371.018084:0:6315:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 00000000c868ccb3. 00040000:00000010:0.0:1773941371.018088:0:6315:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 000000008f2d9c17. 00040000:00000010:0.0:1773941371.018091:0:6315:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000bc3b7c61. 00000004:00000010:0.0:1773941371.018094:0:6315:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 0000000046c79167. 00000004:00000010:0.0:1773941371.018100:0:6315:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at 00000000acaf9781. 00000004:00000010:0.0:1773941371.018106:0:6315:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 0000000021b5a3c9. 00002000:00000010:0.0:1773941371.018110:0:6315:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 0000000086b9feb8. 00000004:00000010:0.0:1773941371.018113:0:6315:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at 000000001075ae7e. 00000020:00000010:0.0:1773941371.018117:0:6315:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 000000007ec150e5. 00000040:00000010:0.0:1773941371.018124:0:6315:0:(llog.c:570:llog_process_thread()) kmalloced '(buf)': 8192 at 000000001d089eff. 00000100:00000010:0.0:1773941371.018138:0:6315:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000bd3dedee. 02000000:00000010:0.0:1773941371.018147:0:6315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 00000000f1c894f7. 00000100:00000010:0.0:1773941371.018156:0:6315:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 00000000182677ba. 02000000:00000010:0.0:1773941371.018175:0:6315:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 16384 at 00000000e30c684f. 00000400:00000200:0.0:1773941371.018180:0:6315:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941371.018182:0:6315:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000cc5e2bc1. 00000400:00000010:0.0:1773941371.018185:0:6315:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 184 at 00000000c02c63ba. 00000100:00000200:0.0:1773941371.018188:0:6315:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 16384 bytes, xid 1860112291464064, portal 25 00000400:00000010:0.0:1773941371.018200:0:6315:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000098ce322e. 00000100:00000200:0.0:1773941371.018202:0:6315:0:(niobuf.c:94:ptl_send_buf()) Sending 272 bytes to portal 26, xid 1860112291464064, offset 0 00000400:00000200:0.0:1773941371.018208:0:6315:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941371.018212:0:6315:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.018216:0:6315:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941371.018220:0:6315:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941371.018222:0:6315:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941371.018225:0:6315:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941371.018227:0:6315:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:0.0:1773941371.018230:0:6315:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [10:10:256:4294967295] 192.168.204.114@tcp LPNI seq info [10:10:8:4294967295] 00000400:00000200:0.0:1773941371.018233:0:6315:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.018240:0:6315:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:0.0:1773941371.018250:0:6315:0:(socklnd_cb.c:1001:ksocknal_send()) sending 272 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:0.0:1773941371.018253:0:6315:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000009cb79a28. 00000800:00000200:0.0:1773941371.018257:0:6315:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006663d74c] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:0.0:1773941371.018262:0:6315:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:0.0:1773941371.018264:0:6315:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 000000009cb79a28 type 1, nob 368 niov 1 nkiov 1 00000400:00000200:0.0:1773941371.018280:0:6315:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:1.0:1773941371.018335:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941371.018339:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000009cb79a28. 00000400:00000200:1.0:1773941371.018344:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941371.018349:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941371.018353:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000098ce322e 00000400:00000010:1.0:1773941371.018355:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 0000000098ce322e. 00000100:00000200:1.0:1773941371.018361:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000bd3dedee x1860112291464064/t0(0) o502->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941387 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1773941371.019240:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.019244:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941371.019248:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.019250:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.019253:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.019255:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 8472 into portal 25 MB=0x69bc324600380 00000400:00000200:0.0:1773941371.019258:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.019261:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 8472/8472 into md 0x271 [4] + 224 00000400:00000200:0.0:1773941371.019273:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941371.019276:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:0.0:1773941371.019281:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000bd3dedee x1860112291464064/t0(0) o502->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941387 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1773941371.019304:0:6315:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 00000000cc5e2bc1. 00000400:00000200:0.0:1773941371.019307:0:6315:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000c02c63ba 00000400:00000010:0.0:1773941371.019309:0:6315:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 184 at 00000000c02c63ba. 00000100:00000200:0.0:1773941371.019312:0:6315:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000bd3dedee x1860112291464064/t0(0) o502->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941387 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941371.019319:0:6315:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000bd3dedee x1860112291464064/t0(0) o502->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941387 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1773941371.019331:0:6315:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at 00000000182677ba. 02000000:00000010:0.0:1773941371.019336:0:6315:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 16384 at 00000000e30c684f. 02000000:00000010:0.0:1773941371.019345:0:6315:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 00000000f1c894f7. 00000100:00000010:0.0:1773941371.019349:0:6315:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000bd3dedee. 00000020:00000010:0.0:1773941371.019367:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 0000000086664f9a. 00000020:00000010:0.0:1773941371.019372:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 0000000086664f9a. 00000020:00000010:0.0:1773941371.019377:0:6315:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 34 at 00000000b5b4bf4c. 00000020:00000010:0.0:1773941371.019380:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 128 at 000000004e41a6a1. 00000020:00000010:0.0:1773941371.019392:0:6315:0:(genops.c:51:obd_device_alloc()) slab-alloced 'obd': 6584 at 00000000b4c49dc6. 00000020:00000010:0.0:1773941371.019398:0:6315:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at 0000000016d03ac8. 00000020:00000010:0.0:1773941371.019414:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 128 at 000000004e41a6a1. 00000020:00000010:0.0:1773941371.019417:0:6315:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 34 at 00000000b5b4bf4c. 00000020:00000010:0.0:1773941371.019420:0:6315:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 34 at 00000000b5b4bf4c. 00000020:00000010:0.0:1773941371.019422:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 160 at 00000000182677ba. 00000001:00000010:0.0:1773941371.019442:0:6315:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at 0000000086664f9a. 00000001:00000010:0.0:1773941371.019447:0:6315:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at 00000000605603ef. 00000001:00000010:0.0:1773941371.019450:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000f1c894f7. 00000001:00000010:0.0:1773941371.019461:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000bfcecc9d. 00000001:00000010:0.0:1773941371.019463:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 0000000017f83e3e. 00000001:00000010:0.0:1773941371.019466:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000d87947cc. 00000020:00000010:0.0:1773941371.019471:0:6315:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000ee18565c. 00000020:00000010:0.0:1773941371.019474:0:6315:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000004e41a6a1. 00000020:00000010:0.0:1773941371.019477:0:6315:0:(tgt_main.c:756:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000002f5aae69. 00000008:00000010:0.0:1773941371.019482:0:6315:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at 00000000ddbd72d3. 00020000:00000010:0.0:1773941371.019485:0:6315:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at 000000000c7f6b29. 00008000:00000010:0.0:1773941371.019489:0:6315:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at 000000003e9d83b2. 00000080:00000010:0.0:1773941371.019493:0:6315:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at 000000001b31c71a. 00000020:00000010:0.0:1773941371.019496:0:6315:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000005313da79. 00000020:00000010:0.0:1773941371.019500:0:6315:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000006f3dac8f. 00000020:00000010:0.0:1773941371.019503:0:6315:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000037845e3. 00000008:00000010:0.0:1773941371.019507:0:6315:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000bbf5a37f. 00020000:00000010:0.0:1773941371.019510:0:6315:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 000000002e187129. 00008000:00000010:0.0:1773941371.019513:0:6315:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 0000000091549415. 00000080:00000010:0.0:1773941371.019517:0:6315:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 00000000197acf3e. 00000080:00000010:0.0:1773941371.019521:0:6315:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 00000000d9255350. 00020000:00000010:0.0:1773941371.019524:0:6315:0:(lov_dev.c:599:lov_device_alloc()) kmalloced '(ld)': 632 at 0000000011b5b19f. 00000020:00000010:0.0:1773941371.019543:0:6315:0:(lu_tgt_pool.c:52:lu_tgt_pool_init()) kmalloced '(op->op_array)': 8 at 00000000dbc91808. 00020000:00000010:0.0:1773941371.019703:0:6315:0:(lov_dev.c:616:lov_device_alloc()) kmalloced '(ld->ld_md_tgts)': 4096 at 000000009813e6bb. 00020000:00000010:0.0:1773941371.019709:0:6315:0:(lov_dev.c:623:lov_device_alloc()) kmalloced '(ld->ld_lov->lov_mdc_tgts)': 4096 at 0000000085e7973e. 00000020:00000010:0.0:1773941371.019754:0:6315:0:(lu_object.c:1154:lu_site_init()) kmalloced '(s->ls_bkts)': 18432 at 00000000e490e6c6. 00000020:00000010:0.0:1773941371.019766:0:6315:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 000000005614f2cf. 00000020:00000010:0.0:1773941371.019771:0:6315:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 192 at 00000000c02c63ba. 00000020:00000010:0.0:1773941371.019779:0:6315:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000c14ac5cd. 00000040:00000010:0.0:1773941371.019783:0:6315:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 000000004d4e6d06. 00000020:00000010:0.0:1773941371.019787:0:6315:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000500e58d8. 00000020:00000010:0.0:1773941371.019793:0:6315:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at 00000000cb6593a5. 00000020:00000010:0.0:1773941371.019797:0:6315:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 000000004b7bd467. 80000000:00000010:0.0:1773941371.019799:0:6315:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 000000004ff90003. 40000000:00000010:0.0:1773941371.019802:0:6315:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 000000003959d60d. 00000008:00000010:0.0:1773941371.019805:0:6315:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 000000005da8e481. 00020000:00000010:0.0:1773941371.019809:0:6315:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 0000000088b69346. 00008000:00000010:0.0:1773941371.019815:0:6315:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 0000000017d45132. 00100000:00000010:0.0:1773941371.019820:0:6315:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 00000000250df6bb. 00040000:00000010:0.0:1773941371.019824:0:6315:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 000000000312ee1f. 00040000:00000010:0.0:1773941371.019828:0:6315:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 000000008fbceff2. 00040000:00000010:0.0:1773941371.019832:0:6315:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 000000009277f046. 00000004:00000010:0.0:1773941371.019836:0:6315:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 000000004aef1fae. 00000004:00000010:0.0:1773941371.019840:0:6315:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at 0000000047257602. 00000004:00000010:0.0:1773941371.019845:0:6315:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 0000000024af87a8. 00002000:00000010:0.0:1773941371.019858:0:6315:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 00000000e30c684f. 00000004:00000010:0.0:1773941371.019862:0:6315:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at 000000008647674d. 00000080:00000010:0.0:1773941371.019867:0:6315:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 00000000bbec8b59. 00000080:00000010:0.0:1773941371.019871:0:6315:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 00000000dc31ffe5. 00000020:00000010:0.0:1773941371.019874:0:6315:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 000000005c876aaf. 00000040:00000010:0.0:1773941371.019878:0:6315:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 0000000063b97608. 00000020:00000010:0.0:1773941371.019882:0:6315:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 000000005cdce0c1. 00000020:00000010:0.0:1773941371.019885:0:6315:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at 00000000b3aaf00a. 00000020:00000010:0.0:1773941371.019888:0:6315:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000091a69c7. 80000000:00000010:0.0:1773941371.019895:0:6315:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 000000003ecd0496. 40000000:00000010:0.0:1773941371.019897:0:6315:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 00000000f5347991. 00000008:00000010:0.0:1773941371.019901:0:6315:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 000000000bcdc5c1. 00020000:00000010:0.0:1773941371.019904:0:6315:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 000000003e8cdd52. 00008000:00000010:0.0:1773941371.019907:0:6315:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 0000000060b832e1. 00100000:00000010:0.0:1773941371.019910:0:6315:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 0000000073ada98e. 00040000:00000010:0.0:1773941371.019912:0:6315:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 000000009cb79a28. 00040000:00000010:0.0:1773941371.019915:0:6315:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 00000000a7beb271. 00040000:00000010:0.0:1773941371.019918:0:6315:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 0000000000f42ca1. 00000004:00000010:0.0:1773941371.019922:0:6315:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 0000000094eeb0e2. 00000004:00000010:0.0:1773941371.019926:0:6315:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at 000000002f2d7528. 00000004:00000010:0.0:1773941371.019929:0:6315:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 00000000f7b3a183. 00002000:00000010:0.0:1773941371.019931:0:6315:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 00000000110a6db4. 00000004:00000010:0.0:1773941371.019934:0:6315:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at 00000000e96cc882. 00000080:00000010:0.0:1773941371.019937:0:6315:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 000000000b3dab9e. 00000080:00000010:0.0:1773941371.019940:0:6315:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 0000000094d66662. 00000020:00000010:0.0:1773941371.019945:0:6315:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 000000006f3dac8f. 00000020:00000010:0.0:1773941371.019949:0:6315:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at 00000000037845e3. 00000008:00000010:0.0:1773941371.019953:0:6315:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at 00000000bbf5a37f. 00020000:00000010:0.0:1773941371.019957:0:6315:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at 000000002e187129. 00008000:00000010:0.0:1773941371.019997:0:6315:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at 0000000091549415. 00000080:00000010:0.0:1773941371.020005:0:6315:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at 00000000197acf3e. 00000080:00000010:0.0:1773941371.020011:0:6315:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at 00000000d9255350. 00000020:00000010:0.0:1773941371.020016:0:6315:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000005313da79. 00000020:00000010:0.0:1773941371.020023:0:6315:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000004e41a6a1. 00000020:00000010:0.0:1773941371.020028:0:6315:0:(tgt_main.c:768:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000002f5aae69. 00000008:00000010:0.0:1773941371.020033:0:6315:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at 00000000ddbd72d3. 00020000:00000010:0.0:1773941371.020039:0:6315:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at 000000000c7f6b29. 00008000:00000010:0.0:1773941371.020044:0:6315:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at 000000003e9d83b2. 00000080:00000010:0.0:1773941371.020049:0:6315:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at 000000001b31c71a. 00000020:00000010:0.0:1773941371.020053:0:6315:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000ee18565c. 00000020:00000010:0.0:1773941371.020059:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 160 at 00000000182677ba. 00000020:00000010:0.0:1773941371.020064:0:6315:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 34 at 00000000b5b4bf4c. 00000020:00000010:0.0:1773941371.020082:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 0000000011b16680. 00000020:00000010:0.0:1773941371.020085:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 0000000011b16680. 00000020:00000010:0.0:1773941371.020091:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 0000000011b16680. 00000020:00000010:0.0:1773941371.020094:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 0000000011b16680. 00000020:00000010:0.0:1773941371.020098:0:6315:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 34 at 00000000b5b4bf4c. 00000020:00000010:0.0:1773941371.020102:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 128 at 000000004e41a6a1. 00000020:00000010:0.0:1773941371.020110:0:6315:0:(genops.c:51:obd_device_alloc()) slab-alloced 'obd': 6584 at 000000000a8b4705. 00000020:00000010:0.0:1773941371.020125:0:6315:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at 0000000063bad8ff. 00000020:00000010:0.0:1773941371.020133:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 128 at 000000004e41a6a1. 00000020:00000010:0.0:1773941371.020136:0:6315:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 34 at 00000000b5b4bf4c. 00000020:00000010:0.0:1773941371.020140:0:6315:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 34 at 00000000b5b4bf4c. 00000020:00000010:0.0:1773941371.020142:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 160 at 00000000182677ba. 00000001:00000010:0.0:1773941371.020153:0:6315:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at 0000000011b16680. 00000001:00000010:0.0:1773941371.020156:0:6315:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at 0000000029b04986. 00000001:00000010:0.0:1773941371.020157:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000ee18565c. 00000001:00000010:0.0:1773941371.020159:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 000000005313da79. 00000001:00000010:0.0:1773941371.020161:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 000000006f3dac8f. 00000001:00000010:0.0:1773941371.020164:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 0000000092cb70f1. 00000020:00000010:0.0:1773941371.020167:0:6315:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000807041a8. 00000020:00000010:0.0:1773941371.020171:0:6315:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000004e41a6a1. 00000020:00000010:0.0:1773941371.020175:0:6315:0:(tgt_main.c:756:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000002f5aae69. 00000008:00000010:0.0:1773941371.020179:0:6315:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at 00000000ddbd72d3. 00020000:00000010:0.0:1773941371.020181:0:6315:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at 000000000c7f6b29. 00008000:00000010:0.0:1773941371.020185:0:6315:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at 000000003e9d83b2. 00000080:00000010:0.0:1773941371.020190:0:6315:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at 000000001b31c71a. 00000020:00000010:0.0:1773941371.020193:0:6315:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000003c40e2fb. 00000020:00000010:0.0:1773941371.020197:0:6315:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000000a7ca015. 00000040:00000010:0.0:1773941371.020210:0:6315:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000ab37d9af. 00000020:00000010:0.0:1773941371.020216:0:6315:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000a2958139. 00000020:00000010:0.0:1773941371.020219:0:6315:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000a89bed9b. 00040000:00000010:0.0:1773941371.020223:0:6315:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 00000000f6cab5dc. 00040000:00000010:0.0:1773941371.020234:0:6315:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000ecee2dbb. 00800000:00000010:0.0:1773941371.020239:0:6315:0:(lmv_obd.c:1219:lmv_device_alloc()) kmalloced '(lu)': 64 at 0000000046051514. 00000020:00000010:0.0:1773941371.020413:0:6315:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000353b66b5. 00000020:00000010:0.0:1773941371.020420:0:6315:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 512 at 000000008769454c. 80000000:00000010:0.0:1773941371.020443:0:6315:0:(fld_cache.c:47:fld_cache_init()) kmalloced '(cache)': 168 at 00000000cd23ae80. 00800000:00000010:0.0:1773941371.020493:0:6315:0:(lmv_obd.c:1279:lmv_device_alloc()) kmalloced '(pat)': 280 at 000000001ccba618. 00800000:00000010:0.0:1773941371.020498:0:6315:0:(lmv_obd.c:1288:lmv_device_alloc()) kmalloced '(pat)': 280 at 00000000c0af1204. 00000020:00000010:0.0:1773941371.020502:0:6315:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 000000000a7ca015. 00000040:00000010:0.0:1773941371.020509:0:6315:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at 00000000ab37d9af. 00000020:00000010:0.0:1773941371.020514:0:6315:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at 00000000a2958139. 00000020:00000010:0.0:1773941371.020518:0:6315:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at 00000000a89bed9b. 00040000:00000010:0.0:1773941371.020522:0:6315:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at 00000000f6cab5dc. 00040000:00000010:0.0:1773941371.020525:0:6315:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at 00000000ecee2dbb. 00000020:00000010:0.0:1773941371.020527:0:6315:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000003c40e2fb. 00000020:00000010:0.0:1773941371.020531:0:6315:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000004e41a6a1. 00000020:00000010:0.0:1773941371.020534:0:6315:0:(tgt_main.c:768:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000002f5aae69. 00000008:00000010:0.0:1773941371.020538:0:6315:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at 00000000ddbd72d3. 00020000:00000010:0.0:1773941371.020543:0:6315:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at 000000000c7f6b29. 00008000:00000010:0.0:1773941371.020547:0:6315:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at 000000003e9d83b2. 00000080:00000010:0.0:1773941371.020551:0:6315:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at 000000001b31c71a. 00000020:00000010:0.0:1773941371.020555:0:6315:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000807041a8. 00000020:00000010:0.0:1773941371.020559:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 160 at 00000000182677ba. 00000020:00000010:0.0:1773941371.020562:0:6315:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 34 at 00000000b5b4bf4c. 00000020:00000010:0.0:1773941371.020567:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 00000000f6cab5dc. 00000020:00000010:0.0:1773941371.020571:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 00000000f6cab5dc. 00000020:00000010:0.0:1773941371.020576:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 00000000f6cab5dc. 00000020:00000010:0.0:1773941371.020578:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 00000000f6cab5dc. 00000020:00000010:0.0:1773941371.020582:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 96 at 00000000037845e3. 00000020:00000010:0.0:1773941371.020586:0:6315:0:(obd_config.c:1186:class_add_profile()) kmalloced '(lprof)': 48 at 00000000b5b4bf4c. 00000020:00000010:0.0:1773941371.020590:0:6315:0:(obd_config.c:1192:class_add_profile()) kmalloced '(lprof->lp_profile)': 14 at 0000000039c9ecd8. 00000020:00000010:0.0:1773941371.020594:0:6315:0:(obd_config.c:1198:class_add_profile()) kmalloced '(lprof->lp_dt)': 14 at 000000001acea3ea. 00000020:00000010:0.0:1773941371.020598:0:6315:0:(obd_config.c:1205:class_add_profile()) kmalloced '(lprof->lp_md)': 14 at 00000000675ff9a0. 00000020:00000010:0.0:1773941371.020607:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 96 at 00000000037845e3. 00000020:00000010:0.0:1773941371.020612:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 00000000f6cab5dc. 00000020:00000010:0.0:1773941371.020614:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 00000000f6cab5dc. 00000020:00000010:0.0:1773941371.020620:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 00000000f6cab5dc. 00000020:00000010:0.0:1773941371.020622:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 00000000f6cab5dc. 00000020:00000010:0.0:1773941371.020627:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 64 at 0000000032b29461. 00000100:00000010:0.0:1773941371.020644:0:6315:0:(lustre_peer.c:71:class_add_uuid()) kmalloced '(data)': 704 at 00000000eb98c785. 00000400:00000200:0.0:1773941371.020650:0:6315:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:0.0:1773941371.020656:0:6315:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.204.114@tcp 00000400:00000200:0.0:1773941371.020660:0:6315:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.020667:0:6315:0:(peer.c:1737:lnet_peer_add()) peer 192.168.204.114@tcp NID flags 0x100001: -17 00000400:00000200:0.0:1773941371.020672:0:6315:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.020679:0:6315:0:(peer.c:2767:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:0.0:1773941371.020683:0:6315:0:(peer.c:2776:lnet_discover_peer_locked()) peer 192.168.204.114@tcp NID 192.168.204.114@tcp: 0. pending discovery 00000400:00000200:0.0:1773941371.020688:0:6315:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.204.114@tcp 00000100:00000010:0.0:1773941371.020692:0:6315:0:(lustre_peer.c:113:class_add_uuid()) kfreed 'data': 704 at 00000000eb98c785. 00000020:00000010:0.0:1773941371.020698:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 64 at 0000000032b29461. 00000020:00000010:0.0:1773941371.020706:0:6315:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 39 at 0000000032b29461. 00000020:00000010:0.0:1773941371.020710:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 136 at 00000000182677ba. 00000020:00000010:0.0:1773941371.020722:0:6315:0:(genops.c:51:obd_device_alloc()) slab-alloced 'obd': 6584 at 000000003e504147. 00000020:00000010:0.0:1773941371.020726:0:6315:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at 00000000a2958139. 00000020:00000010:0.0:1773941371.020737:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 136 at 00000000182677ba. 00000020:00000010:0.0:1773941371.020742:0:6315:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 39 at 0000000032b29461. 00000020:00000010:0.0:1773941371.020750:0:6315:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 39 at 0000000032b29461. 00000020:00000010:0.0:1773941371.020755:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 136 at 00000000182677ba. 00000001:00000010:0.0:1773941371.020783:0:6315:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at 00000000f6cab5dc. 00000001:00000010:0.0:1773941371.020787:0:6315:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at 00000000c1799a9b. 00000001:00000010:0.0:1773941371.020790:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000ecee2dbb. 00000001:00000010:0.0:1773941371.020793:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000a89bed9b. 00000001:00000010:0.0:1773941371.020795:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000ab37d9af. 00000001:00000010:0.0:1773941371.020799:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000f6cfcb61. 00000020:00000010:0.0:1773941371.020803:0:6315:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000c0ec47bd. 00000020:00000010:0.0:1773941371.020810:0:6315:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000004e41a6a1. 00000020:00000010:0.0:1773941371.020813:0:6315:0:(tgt_main.c:756:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000002f5aae69. 00000008:00000010:0.0:1773941371.020816:0:6315:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at 00000000ddbd72d3. 00020000:00000010:0.0:1773941371.020818:0:6315:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at 000000000c7f6b29. 00008000:00000010:0.0:1773941371.020821:0:6315:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at 000000003e9d83b2. 00000080:00000010:0.0:1773941371.020824:0:6315:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at 000000001b31c71a. 00000020:00000010:0.0:1773941371.020828:0:6315:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000e36de3e4. 00000020:00000010:0.0:1773941371.020833:0:6315:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000563672e8. 00000020:00000010:0.0:1773941371.020836:0:6315:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000037845e3. 00000008:00000010:0.0:1773941371.020839:0:6315:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000bbf5a37f. 00020000:00000010:0.0:1773941371.020841:0:6315:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 000000002e187129. 00008000:00000010:0.0:1773941371.020845:0:6315:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 0000000091549415. 00000080:00000010:0.0:1773941371.020848:0:6315:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 00000000197acf3e. 00000080:00000010:0.0:1773941371.020851:0:6315:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 00000000d9255350. 00000002:00000010:0.0:1773941371.020855:0:6315:0:(mdc_dev.c:1726:mdc_device_alloc()) kmalloced '(osc)': 104 at 00000000e4546a82. 00010000:00000010:0.0:1773941371.020868:0:6315:0:(ldlm_lib.c:421:client_obd_setup()) kmalloced '(cli->cl_read_io_latency_by_size)': 3920 at 00000000e46970ed. 00010000:00000010:0.0:1773941371.020875:0:6315:0:(ldlm_lib.c:434:client_obd_setup()) kmalloced '(cli->cl_write_io_latency_by_size)': 3920 at 00000000017a56d1. 00010000:00000010:0.0:1773941371.020880:0:6315:0:(ldlm_lib.c:522:client_obd_setup()) kmalloced '(cli->cl_mod_tag_bitmap)': 64 at 0000000078944115. 00000020:00000010:0.0:1773941371.020886:0:6315:0:(genops.c:1283:class_new_import()) kmalloced '(imp)': 1712 at 000000005d5b8e8d. 00000100:00000200:0.0:1773941371.020907:0:6315:0:(events.c:560:ptlrpc_uuid_to_peer()) 192.168.204.114@tcp->12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941371.020911:0:6315:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.020916:0:6315:0:(peer.c:1471:LNetPrimaryNID()) NID 192.168.204.114@tcp primary NID 192.168.204.114@tcp rc 0 00010000:00000010:0.0:1773941371.020922:0:6315:0:(ldlm_lib.c:70:import_set_conn()) kmalloced '(imp_conn)': 88 at 000000008c104536. 00010000:00000010:0.0:1773941371.020929:0:6315:0:(ldlm_resource.c:955:ldlm_namespace_new()) kmalloced '(ns)': 848 at 00000000dccacd8f. 00000001:00000010:0.0:1773941371.020933:0:6315:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 280 at 00000000500f53fd. 00000001:00000010:0.0:1773941371.020938:0:6315:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 256 at 00000000e176c8dc. 00000001:00000010:0.0:1773941371.021236:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000a0efd00e. 00000001:00000010:0.0:1773941371.021308:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000006d3a8d55. 00000001:00000010:0.0:1773941371.021363:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 0000000076e47e85. 00000001:00000010:0.0:1773941371.021415:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000cede1a04. 00000001:00000010:0.0:1773941371.021474:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000002cdcec14. 00000001:00000010:0.0:1773941371.021530:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000b4cc7450. 00000001:00000010:0.0:1773941371.021589:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000004022dc20. 00000001:00000010:0.0:1773941371.021668:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000014b2a9c. 00000001:00000010:0.0:1773941371.021728:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000a86863a8. 00000001:00000010:0.0:1773941371.021786:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 0000000003723af4. 00000001:00000010:0.0:1773941371.021842:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000a2dfe5cc. 00000001:00000010:0.0:1773941371.021894:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000e9dde4cd. 00000001:00000010:0.0:1773941371.021949:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000003542b280. 00000100:00000010:0.0:1773941371.022023:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000009574c029. 00000100:00000010:1.0:1773941371.022026:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 000000001d24849f. 00000001:00000010:0.0:1773941371.022069:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000468bb892. 00000001:00000010:0.0:1773941371.022121:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000163b3969. 00000001:00000010:0.0:1773941371.022173:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000892fb105. 00000001:00000010:0.0:1773941371.022225:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 0000000014c1af52. 00000001:00000010:0.0:1773941371.022275:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000004a11536c. 00000001:00000010:0.0:1773941371.022327:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000a50630be. 00000001:00000010:0.0:1773941371.022376:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000d8105830. 00000001:00000010:0.0:1773941371.022424:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000ad000411. 00000001:00000010:0.0:1773941371.022476:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000c3315f9e. 00000001:00000010:0.0:1773941371.022524:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000c015c5cb. 00000001:00000010:0.0:1773941371.022572:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000d0482dd3. 00000001:00000010:0.0:1773941371.022629:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 0000000034746833. 00000001:00000010:0.0:1773941371.022703:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000009591abfc. 00000001:00000010:0.0:1773941371.022753:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000e2b838b5. 00000001:00000010:0.0:1773941371.022806:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000ae1d6f3a. 00000001:00000010:0.0:1773941371.022864:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000002d7e2dfd. 00000001:00000010:0.0:1773941371.022917:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000f9b2b2b9. 00000001:00000010:0.0:1773941371.022995:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000d31bbecc. 00000001:00000010:0.0:1773941371.023059:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 0000000032931253. 00000020:00000010:0.0:1773941371.023155:0:6315:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000b4b4da14. 00000020:00000010:0.0:1773941371.023158:0:6315:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 96 at 0000000059bf5767. 00000020:00000010:0.0:1773941371.023210:0:6315:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 000000004925dbaa. 00000020:00000010:0.0:1773941371.023214:0:6315:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 352 at 0000000035f2f9e2. 00000020:00000010:0.0:1773941371.023425:0:6315:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 0000000035240d7d. 00000020:00000010:0.0:1773941371.023429:0:6315:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 512 at 00000000df2854f1. 00000020:00000010:0.0:1773941371.023446:0:6315:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000fd255943. 00000020:00000010:0.0:1773941371.023449:0:6315:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 3680 at 0000000020976abb. 00000040:00000010:0.0:1773941371.023473:0:6315:0:(llog_obd.c:26:llog_new_ctxt()) kmalloced '(ctxt)': 168 at 00000000baebb2e2. 00000002:00000010:0.0:1773941371.023481:0:6315:0:(mdc_changelog.c:816:mdc_changelog_cdev_init()) kmalloced '(entry)': 1304 at 00000000e3c91700. 00000020:00000010:0.0:1773941371.023733:0:6315:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 00000000563672e8. 00000020:00000010:0.0:1773941371.023737:0:6315:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at 00000000037845e3. 00000008:00000010:0.0:1773941371.023741:0:6315:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at 00000000bbf5a37f. 00020000:00000010:0.0:1773941371.023745:0:6315:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at 000000002e187129. 00008000:00000010:0.0:1773941371.023748:0:6315:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at 0000000091549415. 00000080:00000010:0.0:1773941371.023752:0:6315:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at 00000000197acf3e. 00000080:00000010:0.0:1773941371.023756:0:6315:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at 00000000d9255350. 00000020:00000010:0.0:1773941371.023760:0:6315:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000e36de3e4. 00000020:00000010:0.0:1773941371.023764:0:6315:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000004e41a6a1. 00000020:00000010:0.0:1773941371.023767:0:6315:0:(tgt_main.c:768:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000002f5aae69. 00000008:00000010:0.0:1773941371.023771:0:6315:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at 00000000ddbd72d3. 00020000:00000010:0.0:1773941371.023774:0:6315:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at 000000000c7f6b29. 00008000:00000010:0.0:1773941371.023777:0:6315:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at 000000003e9d83b2. 00000080:00000010:0.0:1773941371.023781:0:6315:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at 000000001b31c71a. 00000020:00000010:0.0:1773941371.023784:0:6315:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000c0ec47bd. 00000020:00000010:0.0:1773941371.023787:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 136 at 00000000182677ba. 00000020:00000010:0.0:1773941371.023791:0:6315:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 39 at 0000000032b29461. 00000020:00000010:0.0:1773941371.023797:0:6315:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 34 at 00000000523e0b1b. 00000020:00000010:0.0:1773941371.023802:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 152 at 00000000182677ba. 00000020:00000010:0.0:1773941371.023808:0:6315:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000c0ec47bd. 00000020:00000010:0.0:1773941371.023811:0:6315:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000e36de3e4. 00000040:00000010:0.0:1773941371.023816:0:6315:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000563672e8. 00000020:00000010:0.0:1773941371.023821:0:6315:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 000000006fddda3a. 00000020:00000010:0.0:1773941371.023825:0:6315:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000ed032161. 00040000:00000010:0.0:1773941371.023829:0:6315:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 000000001050b028. 00040000:00000010:0.0:1773941371.023833:0:6315:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 000000005d1b6275. 00800000:00000010:0.0:1773941371.023850:0:6315:0:(lmv_obd.c:433:lmv_add_target()) kmalloced '(tgt)': 368 at 00000000476f6154. 00020000:00000010:0.0:1773941371.023855:0:6315:0:(lu_tgt_descs.c:371:ltd_add_tgt()) kmalloced '(ltd->ltd_tgt_idx[index / (((1UL) << 12) / sizeof(void *))])': 4096 at 000000007638d44b. 00000020:00000010:0.0:1773941371.023859:0:6315:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 00000000e36de3e4. 00000040:00000010:0.0:1773941371.023863:0:6315:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at 00000000563672e8. 00000020:00000010:0.0:1773941371.023866:0:6315:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at 000000006fddda3a. 00000020:00000010:0.0:1773941371.023870:0:6315:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at 00000000ed032161. 00040000:00000010:0.0:1773941371.023874:0:6315:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at 000000001050b028. 00040000:00000010:0.0:1773941371.023877:0:6315:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at 000000005d1b6275. 00000020:00000010:0.0:1773941371.023879:0:6315:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000c0ec47bd. 00000020:00000010:0.0:1773941371.023884:0:6315:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000c0ec47bd. 00000020:00000010:0.0:1773941371.023888:0:6315:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000005d1b6275. 00000020:00000010:0.0:1773941371.023892:0:6315:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000c2b93bd3. 00000008:00000010:0.0:1773941371.023896:0:6315:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000bbf5a37f. 00020000:00000010:0.0:1773941371.023900:0:6315:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 000000002e187129. 00008000:00000010:0.0:1773941371.023903:0:6315:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 0000000091549415. 00000080:00000010:0.0:1773941371.023907:0:6315:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 00000000197acf3e. 00000080:00000010:0.0:1773941371.023910:0:6315:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 00000000d9255350. 00000020:00000010:0.0:1773941371.023919:0:6315:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 000000005d1b6275. 00000020:00000010:0.0:1773941371.023922:0:6315:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at 00000000c2b93bd3. 00000008:00000010:0.0:1773941371.023925:0:6315:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at 00000000bbf5a37f. 00020000:00000010:0.0:1773941371.023928:0:6315:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at 000000002e187129. 00008000:00000010:0.0:1773941371.023931:0:6315:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at 0000000091549415. 00000080:00000010:0.0:1773941371.023934:0:6315:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at 00000000197acf3e. 00000080:00000010:0.0:1773941371.023937:0:6315:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at 00000000d9255350. 00000020:00000010:0.0:1773941371.023939:0:6315:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000c0ec47bd. 00000020:00000010:0.0:1773941371.023941:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 152 at 00000000182677ba. 00000020:00000010:0.0:1773941371.023943:0:6315:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 34 at 00000000523e0b1b. 00000020:00000010:0.0:1773941371.023947:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 000000001050b028. 00000020:00000010:0.0:1773941371.023949:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 000000001050b028. 00000020:00000010:0.0:1773941371.023955:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 000000001050b028. 00000020:00000010:0.0:1773941371.023957:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 000000001050b028. 00000020:00000010:0.0:1773941371.024028:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 000000001050b028. 00000020:00000010:0.0:1773941371.024038:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 000000001050b028. 00000020:00000010:0.0:1773941371.024045:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 64 at 00000000523e0b1b. 00000100:00000010:0.0:1773941371.024050:0:6315:0:(lustre_peer.c:71:class_add_uuid()) kmalloced '(data)': 704 at 0000000079994fe8. 00000400:00000200:0.0:1773941371.024055:0:6315:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:0.0:1773941371.024060:0:6315:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.204.114@tcp 00000400:00000200:0.0:1773941371.024066:0:6315:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.024071:0:6315:0:(peer.c:1737:lnet_peer_add()) peer 192.168.204.114@tcp NID flags 0x100001: -17 00000400:00000200:0.0:1773941371.024077:0:6315:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.024085:0:6315:0:(peer.c:2767:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:0.0:1773941371.024094:0:6315:0:(peer.c:2776:lnet_discover_peer_locked()) peer 192.168.204.114@tcp NID 192.168.204.114@tcp: 0. pending discovery 00000400:00000200:0.0:1773941371.024097:0:6315:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.204.114@tcp 00000100:00000010:0.0:1773941371.024100:0:6315:0:(lustre_peer.c:113:class_add_uuid()) kfreed 'data': 704 at 0000000079994fe8. 00000020:00000010:0.0:1773941371.024105:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 64 at 00000000523e0b1b. 00000020:00000010:0.0:1773941371.024110:0:6315:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 39 at 00000000523e0b1b. 00000020:00000010:0.0:1773941371.024115:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 136 at 00000000182677ba. 00000020:00000010:0.0:1773941371.024135:0:6315:0:(genops.c:51:obd_device_alloc()) slab-alloced 'obd': 6584 at 00000000d213ad88. 00000020:00000010:0.0:1773941371.024142:0:6315:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at 000000006fddda3a. 00000020:00000010:0.0:1773941371.024156:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 136 at 00000000182677ba. 00000020:00000010:0.0:1773941371.024160:0:6315:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 39 at 00000000523e0b1b. 00000020:00000010:0.0:1773941371.024166:0:6315:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 39 at 00000000523e0b1b. 00000020:00000010:0.0:1773941371.024172:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 136 at 00000000182677ba. 00000001:00000010:0.0:1773941371.024199:0:6315:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at 000000001050b028. 00000001:00000010:0.0:1773941371.024204:0:6315:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at 00000000f31dda46. 00000001:00000010:0.0:1773941371.024206:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000c0ec47bd. 00000001:00000010:0.0:1773941371.024209:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 000000005d1b6275. 00000001:00000010:0.0:1773941371.024213:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000ed032161. 00000001:00000010:0.0:1773941371.024216:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000563672e8. 00000020:00000010:0.0:1773941371.024220:0:6315:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000e36de3e4. 00000020:00000010:0.0:1773941371.024223:0:6315:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000004e41a6a1. 00000020:00000010:0.0:1773941371.024226:0:6315:0:(tgt_main.c:756:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000002f5aae69. 00000008:00000010:0.0:1773941371.024231:0:6315:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at 00000000ddbd72d3. 00020000:00000010:0.0:1773941371.024235:0:6315:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at 000000000c7f6b29. 00008000:00000010:0.0:1773941371.024248:0:6315:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at 000000003e9d83b2. 00000080:00000010:0.0:1773941371.024253:0:6315:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at 000000001b31c71a. 00000020:00000010:0.0:1773941371.024257:0:6315:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000807041a8. 00000020:00000010:0.0:1773941371.024260:0:6315:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 000000003c40e2fb. 00000020:00000010:0.0:1773941371.024264:0:6315:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000c2b93bd3. 00000008:00000010:0.0:1773941371.024267:0:6315:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000bbf5a37f. 00020000:00000010:0.0:1773941371.024270:0:6315:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 000000002e187129. 00008000:00000010:0.0:1773941371.024273:0:6315:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 0000000091549415. 00000080:00000010:0.0:1773941371.024277:0:6315:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 00000000197acf3e. 00000080:00000010:0.0:1773941371.024281:0:6315:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 00000000d9255350. 00000002:00000010:0.0:1773941371.024285:0:6315:0:(mdc_dev.c:1726:mdc_device_alloc()) kmalloced '(osc)': 104 at 00000000cbfe9690. 00010000:00000010:0.0:1773941371.024295:0:6315:0:(ldlm_lib.c:421:client_obd_setup()) kmalloced '(cli->cl_read_io_latency_by_size)': 3920 at 0000000094ac4977. 00010000:00000010:0.0:1773941371.024304:0:6315:0:(ldlm_lib.c:434:client_obd_setup()) kmalloced '(cli->cl_write_io_latency_by_size)': 3920 at 00000000ed5abc22. 00010000:00000010:0.0:1773941371.024315:0:6315:0:(ldlm_lib.c:522:client_obd_setup()) kmalloced '(cli->cl_mod_tag_bitmap)': 64 at 00000000ef61e6d1. 00000020:00000010:0.0:1773941371.024320:0:6315:0:(genops.c:1283:class_new_import()) kmalloced '(imp)': 1712 at 00000000006c8732. 00000100:00000200:0.0:1773941371.024333:0:6315:0:(events.c:560:ptlrpc_uuid_to_peer()) 192.168.204.114@tcp->12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941371.024337:0:6315:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.024343:0:6315:0:(peer.c:1471:LNetPrimaryNID()) NID 192.168.204.114@tcp primary NID 192.168.204.114@tcp rc 0 00010000:00000010:0.0:1773941371.024348:0:6315:0:(ldlm_lib.c:70:import_set_conn()) kmalloced '(imp_conn)': 88 at 0000000038a0a1dc. 00010000:00000010:0.0:1773941371.024352:0:6315:0:(ldlm_resource.c:955:ldlm_namespace_new()) kmalloced '(ns)': 848 at 0000000007f32ee0. 00000001:00000010:0.0:1773941371.024356:0:6315:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 280 at 000000000a7ca015. 00000001:00000010:0.0:1773941371.024361:0:6315:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 256 at 000000000c00bac7. 00000001:00000010:0.0:1773941371.024414:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000003552e0cd. 00000001:00000010:0.0:1773941371.024484:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000e30d5726. 00000001:00000010:0.0:1773941371.024552:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000004b0bbee2. 00000001:00000010:0.0:1773941371.024628:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000e75d270f. 00000001:00000010:0.0:1773941371.024708:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000000fe9abec. 00000001:00000010:0.0:1773941371.024774:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 0000000013924f4b. 00000001:00000010:0.0:1773941371.024832:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000f0f2ec82. 00000001:00000010:0.0:1773941371.024893:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000bd197a56. 00000001:00000010:0.0:1773941371.024955:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000e069e512. 00000001:00000010:0.0:1773941371.025058:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000001a3f4d3c. 00000001:00000010:0.0:1773941371.025111:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 0000000079a0e315. 00000001:00000010:0.0:1773941371.025161:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000525cd40e. 00000001:00000010:0.0:1773941371.025223:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000f5f0bc1b. 00000001:00000010:0.0:1773941371.025274:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000b87c6ed8. 00000001:00000010:0.0:1773941371.025343:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000c61f659c. 00000001:00000010:0.0:1773941371.025425:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000005d8b44d0. 00000001:00000010:0.0:1773941371.025498:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000666f6a82. 00000001:00000010:0.0:1773941371.025561:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000773b465f. 00000001:00000010:0.0:1773941371.025616:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 0000000073a3afd1. 00000001:00000010:0.0:1773941371.025689:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000006b3344ce. 00000001:00000010:0.0:1773941371.025746:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 0000000044654444. 00000001:00000010:0.0:1773941371.025814:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000000c50da2b. 00000001:00000010:0.0:1773941371.025876:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000ecb0ef00. 00000001:00000010:0.0:1773941371.025936:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000008bce57de. 00000001:00000010:0.0:1773941371.026015:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000003c385a7a. 00000001:00000010:0.0:1773941371.026062:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000a7ebd532. 00000001:00000010:0.0:1773941371.026105:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000962ea3dc. 00000001:00000010:0.0:1773941371.026159:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000a9722b3b. 00000001:00000010:0.0:1773941371.026209:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 0000000062138cd4. 00000001:00000010:0.0:1773941371.026271:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 00000000a57e2003. 00000001:00000010:0.0:1773941371.026319:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000006a87cbc5. 00000001:00000010:0.0:1773941371.026358:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at 000000001d5a2bee. 00000020:00000010:0.0:1773941371.026436:0:6315:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000f2053afd. 00000020:00000010:0.0:1773941371.026439:0:6315:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 96 at 000000003e66014a. 00000020:00000010:0.0:1773941371.026482:0:6315:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000b61747e4. 00000020:00000010:0.0:1773941371.026492:0:6315:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 352 at 0000000010740453. 00000020:00000010:0.0:1773941371.026691:0:6315:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000c7dab97e. 00000020:00000010:0.0:1773941371.026695:0:6315:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 512 at 000000006d194338. 00000020:00000010:0.0:1773941371.026717:0:6315:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000044fa842. 00000020:00000010:0.0:1773941371.026720:0:6315:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 3680 at 00000000247592ea. 00000040:00000010:0.0:1773941371.026746:0:6315:0:(llog_obd.c:26:llog_new_ctxt()) kmalloced '(ctxt)': 168 at 00000000dcaf4b31. 00000002:00000010:0.0:1773941371.026755:0:6315:0:(mdc_changelog.c:816:mdc_changelog_cdev_init()) kmalloced '(entry)': 1304 at 000000009a71e067. 00000020:00000010:0.0:1773941371.026887:0:6315:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 000000003c40e2fb. 00000020:00000010:0.0:1773941371.026892:0:6315:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at 00000000c2b93bd3. 00000008:00000010:0.0:1773941371.026895:0:6315:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at 00000000bbf5a37f. 00020000:00000010:0.0:1773941371.026899:0:6315:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at 000000002e187129. 00008000:00000010:0.0:1773941371.026905:0:6315:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at 0000000091549415. 00000080:00000010:0.0:1773941371.026911:0:6315:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at 00000000197acf3e. 00000080:00000010:0.0:1773941371.026917:0:6315:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at 00000000d9255350. 00000020:00000010:0.0:1773941371.026923:0:6315:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000807041a8. 00000020:00000010:0.0:1773941371.026929:0:6315:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000004e41a6a1. 00000020:00000010:0.0:1773941371.026935:0:6315:0:(tgt_main.c:768:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000002f5aae69. 00000008:00000010:0.0:1773941371.026939:0:6315:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at 00000000ddbd72d3. 00020000:00000010:0.0:1773941371.026941:0:6315:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at 000000000c7f6b29. 00008000:00000010:0.0:1773941371.026943:0:6315:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at 000000003e9d83b2. 00000080:00000010:0.0:1773941371.026945:0:6315:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at 000000001b31c71a. 00000020:00000010:0.0:1773941371.026947:0:6315:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000e36de3e4. 00000020:00000010:0.0:1773941371.026951:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 136 at 00000000182677ba. 00000020:00000010:0.0:1773941371.026954:0:6315:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 39 at 00000000523e0b1b. 00000020:00000010:0.0:1773941371.027001:0:6315:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 34 at 0000000035870493. 00000020:00000010:0.0:1773941371.027007:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 152 at 000000008c96e925. 00000020:00000010:0.0:1773941371.027013:0:6315:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000aa426fef. 00000020:00000010:0.0:1773941371.027018:0:6315:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000e491df39. 00000040:00000010:0.0:1773941371.027023:0:6315:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000d9805cd3. 00000020:00000010:0.0:1773941371.027028:0:6315:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 0000000011dfeeed. 00000020:00000010:0.0:1773941371.027032:0:6315:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 000000007f05ac82. 00040000:00000010:0.0:1773941371.027035:0:6315:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 000000009b38e124. 00040000:00000010:0.0:1773941371.027041:0:6315:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 0000000073b0f79b. 00800000:00000010:0.0:1773941371.027052:0:6315:0:(lmv_obd.c:433:lmv_add_target()) kmalloced '(tgt)': 368 at 000000009a32d42e. 00000020:00000010:0.0:1773941371.027056:0:6315:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 00000000e491df39. 00000040:00000010:0.0:1773941371.027058:0:6315:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at 00000000d9805cd3. 00000020:00000010:0.0:1773941371.027061:0:6315:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at 0000000011dfeeed. 00000020:00000010:0.0:1773941371.027063:0:6315:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at 000000007f05ac82. 00040000:00000010:0.0:1773941371.027070:0:6315:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at 000000009b38e124. 00040000:00000010:0.0:1773941371.027073:0:6315:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at 0000000073b0f79b. 00000020:00000010:0.0:1773941371.027074:0:6315:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000aa426fef. 00000020:00000010:0.0:1773941371.027078:0:6315:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000aa426fef. 00000020:00000010:0.0:1773941371.027079:0:6315:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 0000000073b0f79b. 00000020:00000010:0.0:1773941371.027081:0:6315:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000c48a2e55. 00000008:00000010:0.0:1773941371.027084:0:6315:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000bbf5a37f. 00020000:00000010:0.0:1773941371.027086:0:6315:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 000000002e187129. 00008000:00000010:0.0:1773941371.027088:0:6315:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 0000000091549415. 00000080:00000010:0.0:1773941371.027091:0:6315:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 00000000197acf3e. 00000080:00000010:0.0:1773941371.027093:0:6315:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 00000000d9255350. 00000020:00000010:0.0:1773941371.027099:0:6315:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 0000000073b0f79b. 00000020:00000010:0.0:1773941371.027101:0:6315:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at 00000000c48a2e55. 00000008:00000010:0.0:1773941371.027103:0:6315:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at 00000000bbf5a37f. 00020000:00000010:0.0:1773941371.027105:0:6315:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at 000000002e187129. 00008000:00000010:0.0:1773941371.027108:0:6315:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at 0000000091549415. 00000080:00000010:0.0:1773941371.027110:0:6315:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at 00000000197acf3e. 00000080:00000010:0.0:1773941371.027113:0:6315:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at 00000000d9255350. 00000020:00000010:0.0:1773941371.027114:0:6315:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000aa426fef. 00000020:00000010:0.0:1773941371.027116:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 152 at 000000008c96e925. 00000020:00000010:0.0:1773941371.027119:0:6315:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 34 at 0000000035870493. 00000020:00000010:0.0:1773941371.027123:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 000000009b38e124. 00000020:00000010:0.0:1773941371.027125:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 000000009b38e124. 00000020:00000010:0.0:1773941371.027130:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 000000009b38e124. 00000020:00000010:0.0:1773941371.027131:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 000000009b38e124. 00000020:00000010:0.0:1773941371.027137:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 000000009b38e124. 00000020:00000010:0.0:1773941371.027139:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 000000009b38e124. 00000020:00000010:0.0:1773941371.027143:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 64 at 0000000035870493. 00000100:00000010:0.0:1773941371.027149:0:6315:0:(lustre_peer.c:71:class_add_uuid()) kmalloced '(data)': 704 at 000000005fbbb1e5. 00000400:00000200:0.0:1773941371.027152:0:6315:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:0.0:1773941371.027156:0:6315:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.204.114@tcp 00000400:00000200:0.0:1773941371.027160:0:6315:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.027163:0:6315:0:(peer.c:1737:lnet_peer_add()) peer 192.168.204.114@tcp NID flags 0x100001: -17 00000400:00000200:0.0:1773941371.027183:0:6315:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.027192:0:6315:0:(peer.c:2767:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:0.0:1773941371.027206:0:6315:0:(peer.c:2776:lnet_discover_peer_locked()) peer 192.168.204.114@tcp NID 192.168.204.114@tcp: 0. pending discovery 00000400:00000200:0.0:1773941371.027216:0:6315:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.204.114@tcp 00000100:00000010:0.0:1773941371.027223:0:6315:0:(lustre_peer.c:113:class_add_uuid()) kfreed 'data': 704 at 000000005fbbb1e5. 00000020:00000010:0.0:1773941371.027232:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 64 at 0000000035870493. 00000020:00000010:0.0:1773941371.027237:0:6315:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 39 at 0000000035870493. 00000020:00000010:0.0:1773941371.027240:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 136 at 000000008c96e925. 00000020:00000010:0.0:1773941371.027250:0:6315:0:(genops.c:51:obd_device_alloc()) slab-alloced 'obd': 6584 at 00000000b45a570c. 00000020:00000010:0.0:1773941371.027257:0:6315:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at 0000000011dfeeed. 00000020:00000010:0.0:1773941371.027274:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 136 at 000000008c96e925. 00000020:00000010:0.0:1773941371.027277:0:6315:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 39 at 0000000035870493. 00000020:00000010:0.0:1773941371.027279:0:6315:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 39 at 0000000035870493. 00000020:00000010:0.0:1773941371.027281:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 136 at 000000008c96e925. 00000001:00000010:0.0:1773941371.027303:0:6315:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at 000000009b38e124. 00000001:00000010:0.0:1773941371.027307:0:6315:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at 0000000075f24d5a. 00000001:00000010:0.0:1773941371.027309:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000aa426fef. 00000001:00000010:0.0:1773941371.027312:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 0000000073b0f79b. 00000001:00000010:0.0:1773941371.027313:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 000000007f05ac82. 00000001:00000010:0.0:1773941371.027320:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000d9805cd3. 00000020:00000010:0.0:1773941371.027323:0:6315:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000e491df39. 00000020:00000010:0.0:1773941371.027326:0:6315:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000004e41a6a1. 00000020:00000010:0.0:1773941371.027330:0:6315:0:(tgt_main.c:756:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000002f5aae69. 00000008:00000010:0.0:1773941371.027333:0:6315:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at 00000000ddbd72d3. 00020000:00000010:0.0:1773941371.027335:0:6315:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at 000000000c7f6b29. 00008000:00000010:0.0:1773941371.027341:0:6315:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at 000000003e9d83b2. 00000080:00000010:0.0:1773941371.027345:0:6315:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at 000000001b31c71a. 00000020:00000010:0.0:1773941371.027348:0:6315:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 0000000055df73e4. 00000020:00000010:0.0:1773941371.027353:0:6315:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000c38e410f. 00000020:00000010:0.0:1773941371.027355:0:6315:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000c48a2e55. 00000008:00000010:0.0:1773941371.027357:0:6315:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000bbf5a37f. 00020000:00000010:0.0:1773941371.027360:0:6315:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 000000002e187129. 00008000:00000010:0.0:1773941371.027370:0:6315:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 0000000091549415. 00000080:00000010:0.0:1773941371.027373:0:6315:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 00000000197acf3e. 00000080:00000010:0.0:1773941371.027375:0:6315:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 00000000d9255350. 00000008:00000010:0.0:1773941371.027378:0:6315:0:(osc_dev.c:192:osc_device_alloc()) kmalloced '(osc)': 104 at 0000000025580d44. 00010000:00000010:0.0:1773941371.027386:0:6315:0:(ldlm_lib.c:421:client_obd_setup()) kmalloced '(cli->cl_read_io_latency_by_size)': 3920 at 00000000ba5160ac. 00010000:00000010:0.0:1773941371.027393:0:6315:0:(ldlm_lib.c:434:client_obd_setup()) kmalloced '(cli->cl_write_io_latency_by_size)': 3920 at 000000000b49e5ea. 00000020:00000010:0.0:1773941371.027404:0:6315:0:(genops.c:1283:class_new_import()) kmalloced '(imp)': 1712 at 000000002318ef8f. 00000100:00000200:0.0:1773941371.027414:0:6315:0:(events.c:560:ptlrpc_uuid_to_peer()) 192.168.204.114@tcp->12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941371.027418:0:6315:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.027424:0:6315:0:(peer.c:1471:LNetPrimaryNID()) NID 192.168.204.114@tcp primary NID 192.168.204.114@tcp rc 0 00010000:00000010:0.0:1773941371.027428:0:6315:0:(ldlm_lib.c:70:import_set_conn()) kmalloced '(imp_conn)': 88 at 0000000057ca70be. 00010000:00000010:0.0:1773941371.027431:0:6315:0:(ldlm_resource.c:955:ldlm_namespace_new()) kmalloced '(ns)': 848 at 00000000d009980d. 00000001:00000010:0.0:1773941371.027434:0:6315:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 280 at 00000000e1241c96. 00000001:00000010:0.0:1773941371.027437:0:6315:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 128 at 00000000f4cce009. 00000001:00000010:0.0:1773941371.027448:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000cb512969. 00000001:00000010:0.0:1773941371.027453:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 000000000b2df43d. 00000001:00000010:0.0:1773941371.027457:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000262b576e. 00000001:00000010:0.0:1773941371.027463:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 000000002a7f7475. 00000001:00000010:0.0:1773941371.027474:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 000000000cf58e94. 00000001:00000010:0.0:1773941371.027479:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000949671a9. 00000001:00000010:0.0:1773941371.027484:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000286909ce. 00000001:00000010:0.0:1773941371.027488:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 0000000003354890. 00000001:00000010:0.0:1773941371.027500:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000169c7cbd. 00000001:00000010:0.0:1773941371.027505:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 000000004b478098. 00000001:00000010:0.0:1773941371.027511:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000479c7342. 00000001:00000010:0.0:1773941371.027516:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000e57ce840. 00000001:00000010:0.0:1773941371.027527:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 000000000b3cf650. 00000001:00000010:0.0:1773941371.027533:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 000000003bc123f1. 00000001:00000010:0.0:1773941371.027537:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000c2430425. 00000001:00000010:0.0:1773941371.027544:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000f9a5be44. 00000020:00000010:0.0:1773941371.027648:0:6315:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 0000000023792353. 00000020:00000010:0.0:1773941371.027656:0:6315:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 96 at 00000000d314236b. 00000020:00000010:0.0:1773941371.027736:0:6315:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000789ae55a. 00000020:00000010:0.0:1773941371.027740:0:6315:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 352 at 00000000f88e5525. 00000020:00000010:0.0:1773941371.028105:0:6315:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 0000000067395bc7. 00000020:00000010:0.0:1773941371.028109:0:6315:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 3680 at 000000000c843665. 00000100:00000010:0.0:1773941371.028154:0:6315:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000bd3dedee. 00000100:00000010:0.0:1773941371.028367:0:6315:0:(client.c:657:ptlrpc_add_rqs_to_pool()) kmalloced '(msg)': 524288 at 00000000fffc32f3. 00000100:00000010:0.0:1773941371.028372:0:6315:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000000832ca96. 00000100:00000010:0.0:1773941371.028538:0:6315:0:(client.c:657:ptlrpc_add_rqs_to_pool()) kmalloced '(msg)': 524288 at 0000000016f92688. 00000100:00000010:0.0:1773941371.028546:0:6315:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000a9847b7a. 00000100:00000010:0.0:1773941371.028705:0:6315:0:(client.c:657:ptlrpc_add_rqs_to_pool()) kmalloced '(msg)': 524288 at 000000001b702ad1. 00000100:00000010:0.0:1773941371.028715:0:6315:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000fa7001c0. 00000100:00000010:0.0:1773941371.028879:0:6315:0:(client.c:657:ptlrpc_add_rqs_to_pool()) kmalloced '(msg)': 524288 at 000000001146e647. 00000100:00000010:0.0:1773941371.028884:0:6315:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000006d0795f3. 00000100:00000010:0.0:1773941371.029079:0:6315:0:(client.c:657:ptlrpc_add_rqs_to_pool()) kmalloced '(msg)': 524288 at 000000001a988a20. 00000100:00000010:0.0:1773941371.029088:0:6315:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000007833658e. 00000100:00000010:0.0:1773941371.029228:0:6315:0:(client.c:657:ptlrpc_add_rqs_to_pool()) kmalloced '(msg)': 524288 at 0000000012522cd8. 00000100:00000010:0.0:1773941371.029235:0:6315:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000d493b67a. 00000100:00000010:0.0:1773941371.029379:0:6315:0:(client.c:657:ptlrpc_add_rqs_to_pool()) kmalloced '(msg)': 524288 at 000000002b088e6c. 00000100:00000010:0.0:1773941371.029384:0:6315:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000f2615c3b. 00000100:00000010:0.0:1773941371.029609:0:6315:0:(client.c:657:ptlrpc_add_rqs_to_pool()) kmalloced '(msg)': 524288 at 000000007396b391. 00000100:00000010:0.0:1773941371.029612:0:6315:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000000bafe907. 00000100:00000010:0.0:1773941371.029834:0:6315:0:(client.c:657:ptlrpc_add_rqs_to_pool()) kmalloced '(msg)': 524288 at 00000000a47ecb26. 00000100:00000010:0.0:1773941371.029839:0:6315:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000d575bf2d. 00000100:00000010:0.0:1773941371.030078:0:6315:0:(client.c:657:ptlrpc_add_rqs_to_pool()) kmalloced '(msg)': 524288 at 00000000944860cb. 00000020:00000010:0.0:1773941371.030084:0:6315:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 00000000c38e410f. 00000020:00000010:0.0:1773941371.030088:0:6315:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at 00000000c48a2e55. 00000008:00000010:0.0:1773941371.030090:0:6315:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at 00000000bbf5a37f. 00020000:00000010:0.0:1773941371.030093:0:6315:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at 000000002e187129. 00008000:00000010:0.0:1773941371.030096:0:6315:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at 0000000091549415. 00000080:00000010:0.0:1773941371.030100:0:6315:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at 00000000197acf3e. 00000080:00000010:0.0:1773941371.030108:0:6315:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at 00000000d9255350. 00000020:00000010:0.0:1773941371.030110:0:6315:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 0000000055df73e4. 00000020:00000010:0.0:1773941371.030112:0:6315:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000004e41a6a1. 00000020:00000010:0.0:1773941371.030115:0:6315:0:(tgt_main.c:768:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000002f5aae69. 00000008:00000010:0.0:1773941371.030119:0:6315:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at 00000000ddbd72d3. 00020000:00000010:0.0:1773941371.030122:0:6315:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at 000000000c7f6b29. 00008000:00000010:0.0:1773941371.030129:0:6315:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at 000000003e9d83b2. 00000080:00000010:0.0:1773941371.030132:0:6315:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at 000000001b31c71a. 00000020:00000010:0.0:1773941371.030134:0:6315:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000e491df39. 00000020:00000010:0.0:1773941371.030139:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 136 at 000000008c96e925. 00000020:00000010:0.0:1773941371.030141:0:6315:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 39 at 0000000035870493. 00000020:00000010:0.0:1773941371.030145:0:6315:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 34 at 0000000035870493. 00000020:00000010:0.0:1773941371.030148:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 120 at 000000004e41a6a1. 00000020:00000010:0.0:1773941371.030156:0:6315:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000e491df39. 00000020:00000010:0.0:1773941371.030159:0:6315:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 0000000055df73e4. 00000020:00000010:0.0:1773941371.030162:0:6315:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000c48a2e55. 00000008:00000010:0.0:1773941371.030164:0:6315:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000bbf5a37f. 00020000:00000010:0.0:1773941371.030166:0:6315:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 000000002e187129. 00008000:00000010:0.0:1773941371.030168:0:6315:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 0000000091549415. 00000080:00000010:0.0:1773941371.030172:0:6315:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 00000000197acf3e. 00000080:00000010:0.0:1773941371.030176:0:6315:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 00000000d9255350. 00020000:00000010:0.0:1773941371.030186:0:6315:0:(lov_obd.c:492:lov_add_target()) kmalloced '(tgt)': 368 at 00000000c38e410f. 00020000:00000010:0.0:1773941371.030191:0:6315:0:(lu_tgt_descs.c:371:ltd_add_tgt()) kmalloced '(ltd->ltd_tgt_idx[index / (((1UL) << 12) / sizeof(void *))])': 4096 at 00000000d8839ab1. 00020000:00000010:0.0:1773941371.030197:0:6315:0:(lov_dev.c:357:lov_expand_targets()) kmalloced '(newd)': 512 at 00000000894c81c9. 00000020:00000010:0.0:1773941371.030199:0:6315:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 0000000055df73e4. 00000020:00000010:0.0:1773941371.030202:0:6315:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at 00000000c48a2e55. 00000008:00000010:0.0:1773941371.030204:0:6315:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at 00000000bbf5a37f. 00020000:00000010:0.0:1773941371.030207:0:6315:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at 000000002e187129. 00008000:00000010:0.0:1773941371.030210:0:6315:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at 0000000091549415. 00000080:00000010:0.0:1773941371.030212:0:6315:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at 00000000197acf3e. 00000080:00000010:0.0:1773941371.030213:0:6315:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at 00000000d9255350. 00000020:00000010:0.0:1773941371.030215:0:6315:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000e491df39. 00000020:00000010:0.0:1773941371.030224:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 120 at 000000004e41a6a1. 00000020:00000010:0.0:1773941371.030227:0:6315:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 34 at 0000000035870493. 00000020:00000010:0.0:1773941371.030232:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 00000000bc8b2d42. 00000020:00000010:0.0:1773941371.030234:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 00000000bc8b2d42. 00000020:00000010:0.0:1773941371.030238:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 00000000bc8b2d42. 00000020:00000010:0.0:1773941371.030239:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 00000000bc8b2d42. 00000020:00000010:0.0:1773941371.030244:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 00000000bc8b2d42. 00000020:00000010:0.0:1773941371.030246:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 00000000bc8b2d42. 00000020:00000010:0.0:1773941371.030249:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 64 at 0000000035870493. 00000100:00000010:0.0:1773941371.030253:0:6315:0:(lustre_peer.c:71:class_add_uuid()) kmalloced '(data)': 704 at 00000000ee32e79b. 00000400:00000200:0.0:1773941371.030256:0:6315:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:0.0:1773941371.030259:0:6315:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.204.114@tcp 00000400:00000200:0.0:1773941371.030264:0:6315:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.030269:0:6315:0:(peer.c:1737:lnet_peer_add()) peer 192.168.204.114@tcp NID flags 0x100001: -17 00000400:00000200:0.0:1773941371.030275:0:6315:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.030279:0:6315:0:(peer.c:2767:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:0.0:1773941371.030281:0:6315:0:(peer.c:2776:lnet_discover_peer_locked()) peer 192.168.204.114@tcp NID 192.168.204.114@tcp: 0. pending discovery 00000400:00000200:0.0:1773941371.030283:0:6315:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.204.114@tcp 00000100:00000010:0.0:1773941371.030285:0:6315:0:(lustre_peer.c:113:class_add_uuid()) kfreed 'data': 704 at 00000000ee32e79b. 00000020:00000010:0.0:1773941371.030287:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 64 at 0000000035870493. 00000020:00000010:0.0:1773941371.030290:0:6315:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 39 at 0000000035870493. 00000020:00000010:0.0:1773941371.030292:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 136 at 000000008c96e925. 00000020:00000010:0.0:1773941371.030299:0:6315:0:(genops.c:51:obd_device_alloc()) slab-alloced 'obd': 6584 at 00000000097df8cd. 00000020:00000010:0.0:1773941371.030303:0:6315:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at 000000003fb96c9d. 00000020:00000010:0.0:1773941371.030310:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 136 at 000000008c96e925. 00000020:00000010:0.0:1773941371.030311:0:6315:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 39 at 0000000035870493. 00000020:00000010:0.0:1773941371.030314:0:6315:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 39 at 0000000035870493. 00000020:00000010:0.0:1773941371.030317:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 136 at 000000008c96e925. 00000001:00000010:0.0:1773941371.030343:0:6315:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at 00000000bc8b2d42. 00000001:00000010:0.0:1773941371.030347:0:6315:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at 000000002666c1cd. 00000001:00000010:0.0:1773941371.030349:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000e491df39. 00000001:00000010:0.0:1773941371.030351:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 0000000055df73e4. 00000001:00000010:0.0:1773941371.030358:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 00000000ae6e76cd. 00000001:00000010:0.0:1773941371.030360:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at 000000006b516e93. 00000020:00000010:0.0:1773941371.030363:0:6315:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000ffe4a797. 00000020:00000010:0.0:1773941371.030366:0:6315:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at 000000004e41a6a1. 00000020:00000010:0.0:1773941371.030368:0:6315:0:(tgt_main.c:756:tgt_ses_key_init()) slab-alloced 'session': 200 at 000000002f5aae69. 00000008:00000010:0.0:1773941371.030371:0:6315:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at 00000000ddbd72d3. 00020000:00000010:0.0:1773941371.030373:0:6315:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at 000000000c7f6b29. 00008000:00000010:0.0:1773941371.030375:0:6315:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at 000000003e9d83b2. 00000080:00000010:0.0:1773941371.030378:0:6315:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at 000000001b31c71a. 00000020:00000010:0.0:1773941371.030380:0:6315:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000e36de3e4. 00000020:00000010:0.0:1773941371.030383:0:6315:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000807041a8. 00000020:00000010:0.0:1773941371.030385:0:6315:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000c48a2e55. 00000008:00000010:0.0:1773941371.030386:0:6315:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000bbf5a37f. 00020000:00000010:0.0:1773941371.030388:0:6315:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 000000002e187129. 00008000:00000010:0.0:1773941371.030390:0:6315:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 0000000091549415. 00000080:00000010:0.0:1773941371.030392:0:6315:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 00000000197acf3e. 00000080:00000010:0.0:1773941371.030393:0:6315:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 00000000d9255350. 00000008:00000010:0.0:1773941371.030395:0:6315:0:(osc_dev.c:192:osc_device_alloc()) kmalloced '(osc)': 104 at 00000000fbbb9d47. 00010000:00000010:0.0:1773941371.030403:0:6315:0:(ldlm_lib.c:421:client_obd_setup()) kmalloced '(cli->cl_read_io_latency_by_size)': 3920 at 00000000cdc69003. 00010000:00000010:0.0:1773941371.030408:0:6315:0:(ldlm_lib.c:434:client_obd_setup()) kmalloced '(cli->cl_write_io_latency_by_size)': 3920 at 0000000000360153. 00000020:00000010:0.0:1773941371.030413:0:6315:0:(genops.c:1283:class_new_import()) kmalloced '(imp)': 1712 at 00000000d9a450ba. 00000100:00000200:0.0:1773941371.030421:0:6315:0:(events.c:560:ptlrpc_uuid_to_peer()) 192.168.204.114@tcp->12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941371.030424:0:6315:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.030429:0:6315:0:(peer.c:1471:LNetPrimaryNID()) NID 192.168.204.114@tcp primary NID 192.168.204.114@tcp rc 0 00010000:00000010:0.0:1773941371.030432:0:6315:0:(ldlm_lib.c:70:import_set_conn()) kmalloced '(imp_conn)': 88 at 0000000063f6fd67. 00010000:00000010:0.0:1773941371.030435:0:6315:0:(ldlm_resource.c:955:ldlm_namespace_new()) kmalloced '(ns)': 848 at 000000001b336514. 00000001:00000010:0.0:1773941371.030439:0:6315:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 280 at 000000003c40e2fb. 00000001:00000010:0.0:1773941371.030442:0:6315:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 128 at 0000000086ac047b. 00000001:00000010:0.0:1773941371.030454:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 0000000016cf13a6. 00000001:00000010:0.0:1773941371.030460:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000e68f38bd. 00000001:00000010:0.0:1773941371.030465:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 000000001d6f4558. 00000001:00000010:0.0:1773941371.030470:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 0000000025198f55. 00000001:00000010:0.0:1773941371.030484:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 0000000080e1cc77. 00000001:00000010:0.0:1773941371.030489:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 000000008843a124. 00000001:00000010:0.0:1773941371.030494:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000cfe059c9. 00000001:00000010:0.0:1773941371.030498:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 0000000070a4d200. 00000001:00000010:0.0:1773941371.030507:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 000000006e30df65. 00000001:00000010:0.0:1773941371.030511:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000050ea5e0. 00000001:00000010:0.0:1773941371.030515:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 000000009b7b4cef. 00000001:00000010:0.0:1773941371.030519:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 0000000040554f77. 00000001:00000010:0.0:1773941371.030527:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 0000000092b08a04. 00000001:00000010:0.0:1773941371.030531:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000af493236. 00000001:00000010:0.0:1773941371.030535:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000fda0e161. 00000001:00000010:0.0:1773941371.030538:0:6315:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at 00000000496e3472. 00000020:00000010:0.0:1773941371.030586:0:6315:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000c6002785. 00000020:00000010:0.0:1773941371.030589:0:6315:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 96 at 0000000050cc50bf. 00000020:00000010:0.0:1773941371.030620:0:6315:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 00000000adc47606. 00000020:00000010:0.0:1773941371.030628:0:6315:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 352 at 0000000015a663cb. 00000020:00000010:0.0:1773941371.030791:0:6315:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at 0000000029624954. 00000020:00000010:0.0:1773941371.030793:0:6315:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 3680 at 00000000fccc6023. 00000020:00000010:0.0:1773941371.030807:0:6315:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 00000000807041a8. 00000020:00000010:0.0:1773941371.030810:0:6315:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at 00000000c48a2e55. 00000008:00000010:0.0:1773941371.030813:0:6315:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at 00000000bbf5a37f. 00020000:00000010:0.0:1773941371.030815:0:6315:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at 000000002e187129. 00008000:00000010:0.0:1773941371.030817:0:6315:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at 0000000091549415. 00000080:00000010:0.0:1773941371.030821:0:6315:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at 00000000197acf3e. 00000080:00000010:0.0:1773941371.030825:0:6315:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at 00000000d9255350. 00000020:00000010:0.0:1773941371.030827:0:6315:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000e36de3e4. 00000020:00000010:0.0:1773941371.030829:0:6315:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at 000000004e41a6a1. 00000020:00000010:0.0:1773941371.030833:0:6315:0:(tgt_main.c:768:tgt_ses_key_fini()) slab-freed '(session)': 200 at 000000002f5aae69. 00000008:00000010:0.0:1773941371.030837:0:6315:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at 00000000ddbd72d3. 00020000:00000010:0.0:1773941371.030839:0:6315:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at 000000000c7f6b29. 00008000:00000010:0.0:1773941371.030845:0:6315:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at 000000003e9d83b2. 00000080:00000010:0.0:1773941371.030854:0:6315:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at 000000001b31c71a. 00000020:00000010:0.0:1773941371.030857:0:6315:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000ffe4a797. 00000020:00000010:0.0:1773941371.030860:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 136 at 000000008c96e925. 00000020:00000010:0.0:1773941371.030862:0:6315:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 39 at 0000000035870493. 00000020:00000010:0.0:1773941371.030865:0:6315:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 34 at 0000000035870493. 00000020:00000010:0.0:1773941371.030868:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 120 at 000000004e41a6a1. 00000020:00000010:0.0:1773941371.030872:0:6315:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000006dc6c2a9. 00000020:00000010:0.0:1773941371.030874:0:6315:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000d2785fe0. 00000020:00000010:0.0:1773941371.030876:0:6315:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000c48a2e55. 00000008:00000010:0.0:1773941371.030879:0:6315:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000bbf5a37f. 00020000:00000010:0.0:1773941371.030883:0:6315:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 000000002e187129. 00008000:00000010:0.0:1773941371.030885:0:6315:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 0000000091549415. 00000080:00000010:0.0:1773941371.030887:0:6315:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 00000000197acf3e. 00000080:00000010:0.0:1773941371.030889:0:6315:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 00000000d9255350. 00020000:00000010:0.0:1773941371.030894:0:6315:0:(lov_obd.c:492:lov_add_target()) kmalloced '(tgt)': 368 at 000000009786e7c6. 00000020:00000010:0.0:1773941371.030896:0:6315:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 00000000d2785fe0. 00000020:00000010:0.0:1773941371.030900:0:6315:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at 00000000c48a2e55. 00000008:00000010:0.0:1773941371.030903:0:6315:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at 00000000bbf5a37f. 00020000:00000010:0.0:1773941371.030905:0:6315:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at 000000002e187129. 00008000:00000010:0.0:1773941371.030907:0:6315:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at 0000000091549415. 00000080:00000010:0.0:1773941371.030910:0:6315:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at 00000000197acf3e. 00000080:00000010:0.0:1773941371.030914:0:6315:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at 00000000d9255350. 00000020:00000010:0.0:1773941371.030916:0:6315:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 000000006dc6c2a9. 00000020:00000010:0.0:1773941371.030918:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 120 at 000000004e41a6a1. 00000020:00000010:0.0:1773941371.030919:0:6315:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 34 at 0000000035870493. 00000020:00000010:0.0:1773941371.030923:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 000000002c4b54a8. 00000020:00000010:0.0:1773941371.030924:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 000000002c4b54a8. 00000020:00000010:0.0:1773941371.030927:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 000000002c4b54a8. 00000020:00000010:0.0:1773941371.030929:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 000000002c4b54a8. 00000020:00000010:0.0:1773941371.030933:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 56 at 0000000035870493. 00000020:00000010:0.0:1773941371.030935:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 56 at 0000000035870493. 00000100:00000010:0.0:1773941371.030945:0:6315:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000084c3e071. 02000000:00000010:0.0:1773941371.030983:0:6315:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 000000006dc6c2a9. 00000100:00000010:0.0:1773941371.031001:0:6315:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 000000008c96e925. 02000000:00000010:0.0:1773941371.031029:0:6315:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 16384 at 0000000077fa6f19. 00000400:00000200:0.0:1773941371.031037:0:6315:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941371.031040:0:6315:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 00000000cc5e2bc1. 00000400:00000010:0.0:1773941371.031046:0:6315:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 184 at 0000000044242749. 00000100:00000200:0.0:1773941371.031054:0:6315:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 16384 bytes, xid 1860112291464192, portal 25 00000400:00000010:0.0:1773941371.031059:0:6315:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000008c150ef4. 00000100:00000200:0.0:1773941371.031061:0:6315:0:(niobuf.c:94:ptl_send_buf()) Sending 272 bytes to portal 26, xid 1860112291464192, offset 0 00000400:00000200:0.0:1773941371.031067:0:6315:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941371.031074:0:6315:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.031079:0:6315:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941371.031083:0:6315:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941371.031086:0:6315:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941371.031089:0:6315:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941371.031093:0:6315:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:0.0:1773941371.031097:0:6315:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [11:11:256:4294967295] 192.168.204.114@tcp LPNI seq info [11:11:8:4294967295] 00000400:00000200:0.0:1773941371.031101:0:6315:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.031109:0:6315:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:0.0:1773941371.031115:0:6315:0:(socklnd_cb.c:1001:ksocknal_send()) sending 272 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:0.0:1773941371.031119:0:6315:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000002c4b54a8. 00000800:00000200:0.0:1773941371.031125:0:6315:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006663d74c] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:0.0:1773941371.031132:0:6315:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:0.0:1773941371.031136:0:6315:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 000000002c4b54a8 type 1, nob 368 niov 1 nkiov 1 00000400:00000200:0.0:1773941371.031154:0:6315:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000010:0.0:1773941371.031158:0:6315:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000c48a2e55. 00000100:00000010:0.0:1773941371.031170:0:6315:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000aa40819a. 00000800:00000200:1.0:1773941371.031408:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941371.031415:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000002c4b54a8. 00000400:00000200:1.0:1773941371.031422:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941371.031430:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941371.031438:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000008c150ef4 00000400:00000010:1.0:1773941371.031441:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 000000008c150ef4. 00000100:00000200:1.0:1773941371.031454:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000084c3e071 x1860112291464192/t0(0) o502->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941387 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1773941371.032569:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.032576:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941371.032585:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.032591:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.032607:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.032611:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 8472 into portal 25 MB=0x69bc324600400 00000400:00000200:0.0:1773941371.032617:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.032626:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 8472/8472 into md 0x281 [4] + 224 00000400:00000200:0.0:1773941371.032668:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941371.032673:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:0.0:1773941371.032682:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000084c3e071 x1860112291464192/t0(0) o502->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941387 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1773941371.033051:0:6315:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 00000000cc5e2bc1. 00000400:00000200:3.0:1773941371.033067:0:6315:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000044242749 00000400:00000010:3.0:1773941371.033070:0:6315:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 184 at 0000000044242749. 00000100:00000200:3.0:1773941371.033090:0:6315:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000084c3e071 x1860112291464192/t0(0) o502->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941387 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941371.033103:0:6315:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000084c3e071 x1860112291464192/t0(0) o502->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941387 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1773941371.033134:0:6315:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at 000000008c96e925. 02000000:00000010:3.0:1773941371.033158:0:6315:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 16384 at 0000000077fa6f19. 02000000:00000010:3.0:1773941371.033174:0:6315:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 000000006dc6c2a9. 00000100:00000010:3.0:1773941371.033180:0:6315:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000084c3e071. 00000020:00000010:3.0:1773941371.033193:0:6315:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 000000000ab1681f. 00000020:00000010:3.0:1773941371.033197:0:6315:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 000000000ab1681f. 00000040:00000010:3.0:1773941371.033203:0:6315:0:(llog.c:891:llog_process_thread()) kvfree 'buf': 8192 at 000000001d089eff. 00000020:00000010:3.0:1773941371.033216:0:6315:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at 000000007ec150e5. 00000020:00000010:3.0:1773941371.033228:0:6315:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 00000000740c71ac. 00000040:00000010:3.0:1773941371.033233:0:6315:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at 0000000090049ec9. 00000020:00000010:3.0:1773941371.033241:0:6315:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at 0000000046233ca4. 00000020:00000010:3.0:1773941371.033247:0:6315:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at 00000000fc35371f. 00000020:00000010:3.0:1773941371.033254:0:6315:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at 00000000661742f1. 80000000:00000010:3.0:1773941371.033259:0:6315:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at 00000000c21f02c8. 40000000:00000010:3.0:1773941371.033264:0:6315:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at 00000000d22667de. 00100000:00000010:3.0:1773941371.033274:0:6315:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at 000000000a654f30. 00040000:00000010:3.0:1773941371.033281:0:6315:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at 00000000c868ccb3. 00040000:00000010:3.0:1773941371.033289:0:6315:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at 000000008f2d9c17. 00040000:00000010:3.0:1773941371.033295:0:6315:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at 00000000bc3b7c61. 00000004:00000010:3.0:1773941371.033303:0:6315:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at 0000000046c79167. 00000004:00000010:3.0:1773941371.033311:0:6315:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at 00000000acaf9781. 00000004:00000010:3.0:1773941371.033319:0:6315:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at 0000000021b5a3c9. 00002000:00000010:3.0:1773941371.033323:0:6315:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at 0000000086b9feb8. 00000004:00000010:3.0:1773941371.033329:0:6315:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at 000000001075ae7e. 00000020:00000010:3.0:1773941371.033335:0:6315:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000d6559bf6. 00000040:00000010:2.0:1773941371.033362:0:6296:0:(llog.c:986:llog_process_or_fork()) kfreed 'lpi': 96 at 00000000984f4721. 00000040:00000010:2.0:1773941371.033375:0:6296:0:(llog.c:70:llog_free_handle()) kvfree 'loghandle->lgh_hdr': 8192 at 000000009ed390fc. 00000040:00000010:2.0:1773941371.033382:0:6296:0:(llog.c:72:llog_free_handle()) kfreed 'loghandle': 352 at 00000000df04275e. 00000020:00000010:2.0:1773941371.033392:0:6296:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 00000000905556b5. 00000040:00000010:2.0:1773941371.033397:0:6296:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at 00000000ebbc95b6. 00000020:00000010:2.0:1773941371.033404:0:6296:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at 00000000fc723563. 00000020:00000010:2.0:1773941371.033408:0:6296:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at 00000000aefdd398. 80000000:00000010:2.0:1773941371.033419:0:6296:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at 00000000862b1b3f. 00040000:00000010:2.0:1773941371.033425:0:6296:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at 00000000479775a9. 20000000:00000010:2.0:1773941371.033432:0:6296:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at 000000008b6f0a19. 00000020:00000010:2.0:1773941371.033449:0:6296:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000e0bb6f67. 10000000:00000010:2.0:1773941371.033453:0:6296:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at 000000008922d770. 00000100:00000010:2.0:1773941371.033483:0:6296:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 000000005b838395. 02000000:00000010:2.0:1773941371.033498:0:6296:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 00000000fa15f48d. 00010000:00000010:2.0:1773941371.033513:0:6296:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at 0000000018d836ca. 00010000:00000010:2.0:1773941371.033520:0:6296:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at 00000000c89d2864. 00000020:00000010:2.0:1773941371.033526:0:6296:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 128 at 0000000060de6226. 00000100:00000010:2.0:1773941371.033542:0:6296:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 00000000b7647805. 02000000:00000010:2.0:1773941371.033559:0:6296:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 0000000083499a9a. 00000400:00000200:2.0:1773941371.033567:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941371.033571:0:6296:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000000bbef44f. 00000400:00000010:2.0:1773941371.033578:0:6296:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000004eebd865. 00000100:00000200:2.0:1773941371.033584:0:6296:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291464320, portal 25 00000400:00000010:2.0:1773941371.033589:0:6296:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 000000008e305335. 00000100:00000200:2.0:1773941371.033593:0:6296:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291464320, offset 0 00000400:00000200:2.0:1773941371.033602:0:6296:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941371.033611:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941371.033622:0:6296:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941371.033629:0:6296:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941371.033648:0:6296:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941371.033654:0:6296:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941371.033658:0:6296:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:2.0:1773941371.033665:0:6296:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [12:12:256:4294967295] 192.168.204.114@tcp LPNI seq info [12:12:8:4294967295] 00000400:00000200:2.0:1773941371.033671:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941371.033687:0:6296:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:2.0:1773941371.033697:0:6296:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:2.0:1773941371.033703:0:6296:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000729f4b1f. 00000800:00000200:2.0:1773941371.033713:0:6296:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006663d74c] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:2.0:1773941371.033726:0:6296:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:2.0:1773941371.033731:0:6296:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 00000000729f4b1f type 1, nob 424 niov 1 nkiov 1 00000400:00000200:2.0:1773941371.033752:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:1.0:1773941371.033829:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941371.033836:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000729f4b1f. 00000400:00000200:1.0:1773941371.033845:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941371.033857:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941371.033861:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000008e305335 00000400:00000010:1.0:1773941371.033864:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 000000008e305335. 00000100:00000200:1.0:1773941371.033874:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@000000005b838395 x1860112291464320/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941387 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1773941371.034608:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.034614:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941371.034621:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.034625:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.034629:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.034632:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 344 into portal 25 MB=0x69bc324600480 00000400:00000200:0.0:1773941371.034649:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.034655:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 344/344 into md 0x291 [1] + 224 00000400:00000200:0.0:1773941371.034669:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941371.034673:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:0.0:1773941371.034681:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@000000005b838395 x1860112291464320/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941387 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1773941371.034737:0:6296:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 000000000bbef44f. 00000400:00000200:3.0:1773941371.034745:0:6296:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 000000004eebd865 00000400:00000010:3.0:1773941371.034747:0:6296:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 000000004eebd865. 00000100:00000200:3.0:1773941371.034756:0:6296:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@000000005b838395 x1860112291464320/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941387 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941371.034767:0:6296:0:(events.c:102:reply_in_callback()) @@@ unlink req@000000005b838395 x1860112291464320/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941387 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1773941371.034785:0:6296:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at 00000000b7647805. 02000000:00000010:3.0:1773941371.034801:0:6296:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 0000000083499a9a. 02000000:00000010:3.0:1773941371.034826:0:6296:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 00000000fa15f48d. 00000100:00000010:3.0:1773941371.034830:0:6296:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 000000005b838395. 10000000:00000010:3.0:1773941371.034840:0:6296:0:(mgc_request.c:1545:mgc_process_recover_log()) kmalloced '(pages)': 2048 at 00000000a8f85e68. 00000100:00000010:3.0:1773941371.035511:0:6296:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000015eb58be. 02000000:00000010:3.0:1773941371.035522:0:6296:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 00000000b5120d88. 00000100:00000010:3.0:1773941371.035531:0:6296:0:(client.c:169:ptlrpc_new_bulk()) kmalloced '(desc)': 984 at 00000000aa518d29. 00000100:00000010:3.0:1773941371.035536:0:6296:0:(client.c:174:ptlrpc_new_bulk()) kmalloced '(desc->bd_vec)': 4096 at 00000000937aa26b. 00000100:00000010:3.0:1773941371.035554:0:6296:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 000000007f4bd093. 00000400:00000200:3.0:1773941371.035569:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941371.035573:0:6296:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000009ef7eb24. 00000400:00000010:3.0:1773941371.035580:0:6296:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 4216 at 00000000fb3be81c. 00000100:00000200:3.0:1773941371.035586:0:6296:0:(niobuf.c:453:ptlrpc_register_bulk()) Setup 1 bulk put-sink buffers: 256 pages 1048576 bytes, mbits x0x69bc324600500-0x69bc324600500, portal 33 02000000:00000010:3.0:1773941371.035592:0:6296:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000e5a90198. 00000400:00000200:3.0:1773941371.035596:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941371.035599:0:6296:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000002847ada8. 00000400:00000010:3.0:1773941371.035602:0:6296:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000039f9e5d. 00000100:00000200:3.0:1773941371.035605:0:6296:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112291464448, portal 25 00000400:00000010:3.0:1773941371.035609:0:6296:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000103b3a63. 00000100:00000200:3.0:1773941371.035612:0:6296:0:(niobuf.c:94:ptl_send_buf()) Sending 304 bytes to portal 26, xid 1860112291464448, offset 0 00000400:00000200:3.0:1773941371.035618:0:6296:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941371.035624:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941371.035629:0:6296:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941371.035632:0:6296:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941371.035653:0:6296:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941371.035656:0:6296:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941371.035658:0:6296:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941371.035662:0:6296:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [13:13:256:4294967295] 192.168.204.114@tcp LPNI seq info [13:13:8:4294967295] 00000400:00000200:3.0:1773941371.035667:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941371.035675:0:6296:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:3.0:1773941371.035681:0:6296:0:(socklnd_cb.c:1001:ksocknal_send()) sending 304 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:3.0:1773941371.035685:0:6296:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 000000000ab1681f. 00000800:00000200:3.0:1773941371.035699:0:6296:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006663d74c] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:3.0:1773941371.035705:0:6296:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:3.0:1773941371.035708:0:6296:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 000000000ab1681f type 1, nob 400 niov 1 nkiov 1 00000400:00000200:3.0:1773941371.035727:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:1.0:1773941371.035798:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941371.035804:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 000000000ab1681f. 00000400:00000200:1.0:1773941371.035809:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941371.035816:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941371.035828:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000103b3a63 00000400:00000010:1.0:1773941371.035830:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000103b3a63. 00000100:00000200:1.0:1773941371.035837:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000015eb58be x1860112291464448/t0(0) o256->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 304/240 e 0 to 0 dl 1773941387 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1773941371.036475:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.036480:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941371.036486:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.036489:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.036491:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.036494:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 176 into portal 33 MB=0x69bc324600500 00000400:00000200:0.0:1773941371.036497:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.036500:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 21 from 12345-192.168.204.114@tcp of length 176/176 into md 0x299 [256] + 0 00000400:00000010:0.0:1773941371.036503:0:2393:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 000000009ef7eb24. 00000400:00000200:0.0:1773941371.036514:0:2393:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md 00000000fb3be81c 00000400:00000200:0.0:1773941371.036524:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941371.036527:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941371.036529:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000fb3be81c 00000400:00000010:0.0:1773941371.036531:0:2393:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 4216 at 00000000fb3be81c. 00000100:00000200:0.0:1773941371.036535:0:2393:0:(events.c:192:client_bulk_callback()) event type 2, status 0, req 0000000015eb58be desc 00000000aa518d29 mbits 1860112291464448 00000400:00000200:0.0:1773941371.036551:0:2393:0:(lib-md.c:96:lnet_cpt_of_md()) Couldn't resolve first page of md 0000000000000000 with offset 0 00000400:00000200:0.0:1773941371.036554:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.036559:0:2393:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source Specified: 192.168.204.14@tcp to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941371.036562:0:2393:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [14:14:256:4294967295] 192.168.204.114@tcp LPNI seq info [14:14:8:4294967295] 00000400:00000200:0.0:1773941371.036566:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941371.036572:0:2393:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:192.168.204.14@tcp) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) 192.168.204.114@tcp : ACK try# 0 00000800:00000200:0.0:1773941371.036579:0:2393:0:(socklnd_cb.c:1001:ksocknal_send()) sending 0 bytes in 256 frags to 12345-192.168.204.114@tcp 00000800:00000010:0.0:1773941371.036595:0:2393:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 4328 at 000000008fc6f271. 00000800:00000200:0.0:1773941371.036601:0:2393:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006663d74c] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:0.0:1773941371.036606:0:2393:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:0.0:1773941371.036609:0:2393:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 000000008fc6f271 type 0, nob 96 niov 1 nkiov 0 00000800:00000200:0.0:1773941371.036668:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941371.036670:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 4328 at 000000008fc6f271. 00000400:00000200:0.0:1773941371.036675:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941371.036679:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: ACK: OK 00000400:00000200:1.0:1773941371.037281:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.037287:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941371.037293:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.037296:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.037300:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.037303:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 240 into portal 25 MB=0x69bc324600500 00000400:00000200:1.0:1773941371.037307:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.037311:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 240/240 into md 0x2a1 [1] + 224 00000400:00000200:1.0:1773941371.037322:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941371.037325:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:1.0:1773941371.037330:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000015eb58be x1860112291464448/t0(0) o256->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 304/240 e 0 to 0 dl 1773941387 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1773941371.037366:0:6296:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 000000002847ada8. 00000400:00000200:3.0:1773941371.037373:0:6296:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000039f9e5d 00000400:00000010:3.0:1773941371.037375:0:6296:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000039f9e5d. 00000100:00000200:3.0:1773941371.037381:0:6296:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000015eb58be x1860112291464448/t0(0) o256->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 304/240 e 0 to 0 dl 1773941387 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941371.037389:0:6296:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000015eb58be x1860112291464448/t0(0) o256->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 304/240 e 0 to 0 dl 1773941387 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1773941371.037403:0:6296:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at 000000007f4bd093. 10000000:00000010:3.0:1773941371.037412:0:6296:0:(mgc_request.c:1297:mgc_apply_recover_logs()) kmalloced '(buf)': 4096 at 0000000048b292b0. 00000100:00000200:3.0:1773941371.037422:0:6296:0:(lustre_peer.c:175:class_add_nids_to_uuid()) Comparing 192.168.204.114@tcp with 192.168.204.114@tcp 00000100:00000200:3.0:1773941371.037424:0:6296:0:(lustre_peer.c:182:class_add_nids_to_uuid()) Updating UUID '192.168.204.114@tcp' 00000400:00000200:3.0:1773941371.037426:0:6296:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:3.0:1773941371.037430:0:6296:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.204.114@tcp 00000400:00000200:3.0:1773941371.037433:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941371.037437:0:6296:0:(peer.c:1737:lnet_peer_add()) peer 192.168.204.114@tcp NID flags 0x100001: -17 00000400:00000200:3.0:1773941371.037441:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941371.037446:0:6296:0:(peer.c:2767:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1773941371.037448:0:6296:0:(peer.c:2776:lnet_discover_peer_locked()) peer 192.168.204.114@tcp NID 192.168.204.114@tcp: 0. pending discovery 00000400:00000200:3.0:1773941371.037451:0:6296:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.204.114@tcp 10000000:00000010:3.0:1773941371.037455:0:6296:0:(mgc_request.c:1479:mgc_apply_recover_logs()) kmalloced '(lcfg)': 128 at 00000000cfdf9460. 00000020:00000010:3.0:1773941371.037462:0:6296:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000bc3b7c61. 00000020:00000010:3.0:1773941371.037466:0:6296:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000103f1750. 00000020:00000010:3.0:1773941371.037468:0:6296:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000e4b17198. 00000008:00000010:3.0:1773941371.037474:0:6296:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000f98c81d5. 00020000:00000010:3.0:1773941371.037478:0:6296:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 000000000cc3ff0f. 00008000:00000010:3.0:1773941371.037483:0:6296:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 00000000e005377b. 00000080:00000010:3.0:1773941371.037488:0:6296:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 0000000071cf9cf2. 00000080:00000010:3.0:1773941371.037492:0:6296:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 00000000d3f3b2c7. 00000020:00000010:3.0:1773941371.037543:0:6296:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 00000000103f1750. 00000020:00000010:3.0:1773941371.037549:0:6296:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at 00000000e4b17198. 00000008:00000010:3.0:1773941371.037554:0:6296:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at 00000000f98c81d5. 00020000:00000010:3.0:1773941371.037559:0:6296:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at 000000000cc3ff0f. 00008000:00000010:3.0:1773941371.037564:0:6296:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at 00000000e005377b. 00000080:00000010:3.0:1773941371.037570:0:6296:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at 0000000071cf9cf2. 00000080:00000010:3.0:1773941371.037573:0:6296:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at 00000000d3f3b2c7. 00000020:00000010:3.0:1773941371.037578:0:6296:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000bc3b7c61. 10000000:00000010:3.0:1773941371.037585:0:6296:0:(mgc_request.c:1495:mgc_apply_recover_logs()) kfreed 'lcfg': 128 at 00000000cfdf9460. 00000100:00000200:3.0:1773941371.037595:0:6296:0:(lustre_peer.c:175:class_add_nids_to_uuid()) Comparing 192.168.204.114@tcp with 192.168.204.114@tcp 00000100:00000200:3.0:1773941371.037598:0:6296:0:(lustre_peer.c:182:class_add_nids_to_uuid()) Updating UUID '192.168.204.114@tcp' 00000400:00000200:3.0:1773941371.037600:0:6296:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:3.0:1773941371.037603:0:6296:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.204.114@tcp 00000400:00000200:3.0:1773941371.037606:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941371.037609:0:6296:0:(peer.c:1737:lnet_peer_add()) peer 192.168.204.114@tcp NID flags 0x100001: -17 00000400:00000200:3.0:1773941371.037614:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941371.037617:0:6296:0:(peer.c:2767:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1773941371.037621:0:6296:0:(peer.c:2776:lnet_discover_peer_locked()) peer 192.168.204.114@tcp NID 192.168.204.114@tcp: 0. pending discovery 00000400:00000200:3.0:1773941371.037623:0:6296:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.204.114@tcp 10000000:00000010:3.0:1773941371.037627:0:6296:0:(mgc_request.c:1479:mgc_apply_recover_logs()) kmalloced '(lcfg)': 128 at 00000000cfdf9460. 00000020:00000010:3.0:1773941371.037632:0:6296:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000103f1750. 00000020:00000010:3.0:1773941371.037651:0:6296:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000d6559bf6. 00000020:00000010:3.0:1773941371.037654:0:6296:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000e4b17198. 00000008:00000010:3.0:1773941371.037658:0:6296:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000f98c81d5. 00020000:00000010:3.0:1773941371.037661:0:6296:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 000000000cc3ff0f. 00008000:00000010:3.0:1773941371.037664:0:6296:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 00000000e005377b. 00000080:00000010:3.0:1773941371.037668:0:6296:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 0000000071cf9cf2. 00000080:00000010:3.0:1773941371.037671:0:6296:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 00000000d3f3b2c7. 00000020:00000010:3.0:1773941371.037700:0:6296:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 00000000d6559bf6. 00000020:00000010:3.0:1773941371.037703:0:6296:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at 00000000e4b17198. 00000008:00000010:3.0:1773941371.037706:0:6296:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at 00000000f98c81d5. 00020000:00000010:3.0:1773941371.037710:0:6296:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at 000000000cc3ff0f. 00008000:00000010:3.0:1773941371.037714:0:6296:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at 00000000e005377b. 00000080:00000010:3.0:1773941371.037724:0:6296:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at 0000000071cf9cf2. 00000080:00000010:3.0:1773941371.037728:0:6296:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at 00000000d3f3b2c7. 00000020:00000010:3.0:1773941371.037730:0:6296:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000103f1750. 10000000:00000010:3.0:1773941371.037733:0:6296:0:(mgc_request.c:1495:mgc_apply_recover_logs()) kfreed 'lcfg': 128 at 00000000cfdf9460. 00000100:00000200:3.0:1773941371.037740:0:6296:0:(lustre_peer.c:175:class_add_nids_to_uuid()) Comparing 192.168.204.114@tcp with 192.168.204.114@tcp 00000100:00000200:3.0:1773941371.037742:0:6296:0:(lustre_peer.c:182:class_add_nids_to_uuid()) Updating UUID '192.168.204.114@tcp' 00000400:00000200:3.0:1773941371.037744:0:6296:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:3.0:1773941371.037746:0:6296:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.204.114@tcp 00000400:00000200:3.0:1773941371.037750:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941371.037756:0:6296:0:(peer.c:1737:lnet_peer_add()) peer 192.168.204.114@tcp NID flags 0x100001: -17 00000400:00000200:3.0:1773941371.037762:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941371.037777:0:6296:0:(peer.c:2767:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1773941371.037785:0:6296:0:(peer.c:2776:lnet_discover_peer_locked()) peer 192.168.204.114@tcp NID 192.168.204.114@tcp: 0. pending discovery 00000400:00000200:3.0:1773941371.037788:0:6296:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.204.114@tcp 10000000:00000010:3.0:1773941371.037791:0:6296:0:(mgc_request.c:1479:mgc_apply_recover_logs()) kmalloced '(lcfg)': 128 at 00000000cfdf9460. 00000020:00000010:3.0:1773941371.037803:0:6296:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000103f1750. 00000020:00000010:3.0:1773941371.037806:0:6296:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000d6559bf6. 00000020:00000010:3.0:1773941371.037808:0:6296:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000e4b17198. 00000008:00000010:3.0:1773941371.037811:0:6296:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000f98c81d5. 00020000:00000010:3.0:1773941371.037812:0:6296:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 000000000cc3ff0f. 00008000:00000010:3.0:1773941371.037814:0:6296:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 00000000e005377b. 00000080:00000010:3.0:1773941371.037817:0:6296:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 0000000071cf9cf2. 00000080:00000010:3.0:1773941371.037818:0:6296:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 00000000d3f3b2c7. 00000020:00000010:3.0:1773941371.037856:0:6296:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 00000000d6559bf6. 00000020:00000010:3.0:1773941371.037859:0:6296:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at 00000000e4b17198. 00000008:00000010:3.0:1773941371.037864:0:6296:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at 00000000f98c81d5. 00020000:00000010:3.0:1773941371.037871:0:6296:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at 000000000cc3ff0f. 00008000:00000010:3.0:1773941371.037874:0:6296:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at 00000000e005377b. 00000080:00000010:3.0:1773941371.037877:0:6296:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at 0000000071cf9cf2. 00000080:00000010:3.0:1773941371.037879:0:6296:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at 00000000d3f3b2c7. 00000020:00000010:3.0:1773941371.037887:0:6296:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000103f1750. 10000000:00000010:3.0:1773941371.037891:0:6296:0:(mgc_request.c:1495:mgc_apply_recover_logs()) kfreed 'lcfg': 128 at 00000000cfdf9460. 00000100:00000200:3.0:1773941371.037898:0:6296:0:(lustre_peer.c:175:class_add_nids_to_uuid()) Comparing 192.168.204.114@tcp with 192.168.204.114@tcp 00000100:00000200:3.0:1773941371.037909:0:6296:0:(lustre_peer.c:182:class_add_nids_to_uuid()) Updating UUID '192.168.204.114@tcp' 00000400:00000200:3.0:1773941371.037912:0:6296:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:3.0:1773941371.037915:0:6296:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.204.114@tcp 00000400:00000200:3.0:1773941371.037920:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941371.037926:0:6296:0:(peer.c:1737:lnet_peer_add()) peer 192.168.204.114@tcp NID flags 0x100001: -17 00000400:00000200:3.0:1773941371.037930:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941371.037936:0:6296:0:(peer.c:2767:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1773941371.037941:0:6296:0:(peer.c:2776:lnet_discover_peer_locked()) peer 192.168.204.114@tcp NID 192.168.204.114@tcp: 0. pending discovery 00000400:00000200:3.0:1773941371.037943:0:6296:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.204.114@tcp 10000000:00000010:3.0:1773941371.037949:0:6296:0:(mgc_request.c:1479:mgc_apply_recover_logs()) kmalloced '(lcfg)': 128 at 00000000cfdf9460. 00000020:00000010:3.0:1773941371.037955:0:6296:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000103f1750. 00000020:00000010:3.0:1773941371.037991:0:6296:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000d6559bf6. 00000020:00000010:3.0:1773941371.037995:0:6296:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at 00000000e4b17198. 00000008:00000010:3.0:1773941371.037999:0:6296:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at 00000000f98c81d5. 00020000:00000010:3.0:1773941371.038001:0:6296:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at 000000000cc3ff0f. 00008000:00000010:3.0:1773941371.038005:0:6296:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at 00000000e005377b. 00000080:00000010:3.0:1773941371.038010:0:6296:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at 0000000071cf9cf2. 00000080:00000010:3.0:1773941371.038012:0:6296:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at 00000000d3f3b2c7. 00000020:00000010:3.0:1773941371.038043:0:6296:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at 00000000d6559bf6. 00000020:00000010:3.0:1773941371.038046:0:6296:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at 00000000e4b17198. 00000008:00000010:3.0:1773941371.038049:0:6296:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at 00000000f98c81d5. 00020000:00000010:3.0:1773941371.038052:0:6296:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at 000000000cc3ff0f. 00008000:00000010:3.0:1773941371.038062:0:6296:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at 00000000e005377b. 00000080:00000010:3.0:1773941371.038065:0:6296:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at 0000000071cf9cf2. 00000080:00000010:3.0:1773941371.038067:0:6296:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at 00000000d3f3b2c7. 00000020:00000010:3.0:1773941371.038073:0:6296:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at 00000000103f1750. 10000000:00000010:3.0:1773941371.038075:0:6296:0:(mgc_request.c:1495:mgc_apply_recover_logs()) kfreed 'lcfg': 128 at 00000000cfdf9460. 10000000:00000010:3.0:1773941371.038079:0:6296:0:(mgc_request.c:1507:mgc_apply_recover_logs()) kfreed 'buf': 4096 at 0000000048b292b0. 02000000:00000010:3.0:1773941371.038085:0:6296:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000e5a90198. 00000100:00000010:3.0:1773941371.038096:0:6296:0:(client.c:333:ptlrpc_free_bulk()) kvfree 'desc->bd_vec': 4096 at 00000000937aa26b. 00000100:00000010:3.0:1773941371.038099:0:6296:0:(client.c:334:ptlrpc_free_bulk()) kfreed 'desc': 984 at 00000000aa518d29. 02000000:00000010:3.0:1773941371.038102:0:6296:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 00000000b5120d88. 00000100:00000010:3.0:1773941371.038107:0:6296:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000015eb58be. 00000100:00000010:0.0:1773941371.039026:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at 00000000c48a2e55. 10000000:00000010:3.0:1773941371.039052:0:6296:0:(mgc_request.c:1663:mgc_process_recover_log()) kvfree 'pages': 2048 at 00000000a8f85e68. 00000100:00000010:3.0:1773941371.039083:0:6296:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000015eb58be. 02000000:00000010:3.0:1773941371.039095:0:6296:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 00000000103f1750. 00010000:00000010:3.0:1773941371.039109:0:6296:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at 00000000317dac47. 00010000:00000010:3.0:1773941371.039116:0:6296:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at 00000000243fd126. 00000020:00000010:3.0:1773941371.039121:0:6296:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 128 at 00000000cfdf9460. 00000100:00000010:3.0:1773941371.039134:0:6296:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 000000007f4bd093. 02000000:00000010:3.0:1773941371.039145:0:6296:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at 00000000aa518d29. 00000400:00000200:3.0:1773941371.039151:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941371.039155:0:6296:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000002847ada8. 00000400:00000010:3.0:1773941371.039160:0:6296:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000039f9e5d. 00000100:00000200:3.0:1773941371.039165:0:6296:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291464576, portal 25 00000400:00000010:3.0:1773941371.039169:0:6296:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000065888bc0. 00000100:00000200:3.0:1773941371.039173:0:6296:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291464576, offset 0 00000400:00000200:3.0:1773941371.039180:0:6296:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941371.039186:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941371.039191:0:6296:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941371.039196:0:6296:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941371.039200:0:6296:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941371.039204:0:6296:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941371.039208:0:6296:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941371.039213:0:6296:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [15:15:256:4294967295] 192.168.204.114@tcp LPNI seq info [15:15:8:4294967295] 00000400:00000200:3.0:1773941371.039218:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941371.039230:0:6296:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:3.0:1773941371.039236:0:6296:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:3.0:1773941371.039240:0:6296:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000437ac190. 00000800:00000200:3.0:1773941371.039246:0:6296:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006663d74c] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:3.0:1773941371.039252:0:6296:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:3.0:1773941371.039256:0:6296:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 00000000437ac190 type 1, nob 424 niov 1 nkiov 1 00000400:00000200:3.0:1773941371.039273:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000010:3.0:1773941371.039277:0:6296:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at 00000000e4b17198. 00000800:00000200:0.0:1773941371.039342:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941371.039346:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000437ac190. 00000400:00000200:0.0:1773941371.039350:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941371.039356:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941371.039362:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000065888bc0 00000400:00000010:0.0:1773941371.039364:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 0000000065888bc0. 00000100:00000200:0.0:1773941371.039371:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000015eb58be x1860112291464576/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941387 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1773941371.040069:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.040075:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941371.040085:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.040088:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.040093:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.040099:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 344 into portal 25 MB=0x69bc324600580 00000400:00000200:1.0:1773941371.040103:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.040107:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 344/344 into md 0x2a9 [1] + 224 00000400:00000200:1.0:1773941371.040119:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941371.040123:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:1.0:1773941371.040128:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000015eb58be x1860112291464576/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941387 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1773941371.040163:0:6296:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 000000002847ada8. 00000400:00000200:3.0:1773941371.040170:0:6296:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000039f9e5d 00000400:00000010:3.0:1773941371.040174:0:6296:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000039f9e5d. 00000100:00000200:3.0:1773941371.040183:0:6296:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000015eb58be x1860112291464576/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941387 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941371.040194:0:6296:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000015eb58be x1860112291464576/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941387 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1773941371.040211:0:6296:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at 000000007f4bd093. 02000000:00000010:3.0:1773941371.040223:0:6296:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at 00000000aa518d29. 02000000:00000010:3.0:1773941371.040226:0:6296:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 00000000103f1750. 00000100:00000010:3.0:1773941371.040230:0:6296:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000015eb58be. 10000000:00000010:3.0:1773941371.040239:0:6296:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at 00000000e9398439. 00000020:00000010:3.0:1773941371.040244:0:6296:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 00000000103f1750. 00000020:00000010:3.0:1773941371.040248:0:6296:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000d6559bf6. 00000040:00000010:3.0:1773941371.040252:0:6296:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000661742f1. 00000020:00000010:3.0:1773941371.040258:0:6296:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 00000000a8f85e68. 00000020:00000010:3.0:1773941371.040265:0:6296:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 0000000090049ec9. 80000000:00000010:3.0:1773941371.040268:0:6296:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 000000007069eb54. 00040000:00000010:3.0:1773941371.040271:0:6296:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000740c71ac. 20000000:00000010:3.0:1773941371.040280:0:6296:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at 00000000b5120d88. 00000040:00000010:3.0:1773941371.040286:0:6296:0:(llog.c:42:llog_alloc_handle()) kmalloced '(loghandle)': 352 at 00000000e5a90198. 00000100:00000010:3.0:1773941371.040294:0:6296:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000015eb58be. 02000000:00000010:3.0:1773941371.040301:0:6296:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 00000000bc3b7c61. 00000100:00000010:3.0:1773941371.040307:0:6296:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 000000007f4bd093. 02000000:00000010:3.0:1773941371.040314:0:6296:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at 00000000a121eb75. 00000400:00000200:3.0:1773941371.040317:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941371.040319:0:6296:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000002847ada8. 00000400:00000010:3.0:1773941371.040322:0:6296:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000039f9e5d. 00000100:00000200:3.0:1773941371.040326:0:6296:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112291464704, portal 25 00000400:00000010:3.0:1773941371.040331:0:6296:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 0000000040f7bdfa. 00000100:00000200:3.0:1773941371.040333:0:6296:0:(niobuf.c:94:ptl_send_buf()) Sending 504 bytes to portal 26, xid 1860112291464704, offset 0 00000400:00000200:3.0:1773941371.040338:0:6296:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941371.040350:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941371.040357:0:6296:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941371.040359:0:6296:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941371.040363:0:6296:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941371.040365:0:6296:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941371.040367:0:6296:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941371.040372:0:6296:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [16:16:256:4294967295] 192.168.204.114@tcp LPNI seq info [16:16:8:4294967295] 00000400:00000200:3.0:1773941371.040376:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941371.040381:0:6296:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:3.0:1773941371.040384:0:6296:0:(socklnd_cb.c:1001:ksocknal_send()) sending 504 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:3.0:1773941371.040387:0:6296:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000437ac190. 00000800:00000200:3.0:1773941371.040391:0:6296:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006663d74c] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:3.0:1773941371.040395:0:6296:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:3.0:1773941371.040397:0:6296:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 00000000437ac190 type 1, nob 600 niov 1 nkiov 1 00000400:00000200:3.0:1773941371.040413:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:0.0:1773941371.040466:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941371.040470:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000437ac190. 00000400:00000200:0.0:1773941371.040473:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941371.040477:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941371.040480:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000040f7bdfa 00000400:00000010:0.0:1773941371.040481:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 0000000040f7bdfa. 00000100:00000200:0.0:1773941371.040486:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000015eb58be x1860112291464704/t0(0) o501->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 504/272 e 0 to 0 dl 1773941387 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1773941371.041113:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.041118:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941371.041127:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.041130:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.041133:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.041136:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 272 into portal 25 MB=0x69bc324600600 00000400:00000200:1.0:1773941371.041139:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.041142:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 272/272 into md 0x2b1 [1] + 224 00000400:00000200:1.0:1773941371.041151:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941371.041155:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:1.0:1773941371.041164:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000015eb58be x1860112291464704/t0(0) o501->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 504/272 e 0 to 0 dl 1773941387 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1773941371.041231:0:6296:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 000000002847ada8. 00000400:00000200:2.0:1773941371.041240:0:6296:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000039f9e5d 00000400:00000010:2.0:1773941371.041242:0:6296:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000039f9e5d. 00000100:00000200:2.0:1773941371.041249:0:6296:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000015eb58be x1860112291464704/t0(0) o501->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 504/272 e 0 to 0 dl 1773941387 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941371.041258:0:6296:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000015eb58be x1860112291464704/t0(0) o501->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 504/272 e 0 to 0 dl 1773941387 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1773941371.041276:0:6296:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at 000000007f4bd093. 02000000:00000010:2.0:1773941371.041282:0:6296:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at 00000000a121eb75. 02000000:00000010:2.0:1773941371.041286:0:6296:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 00000000bc3b7c61. 00000100:00000010:2.0:1773941371.041289:0:6296:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000015eb58be. 00000040:00000010:2.0:1773941371.041297:0:6296:0:(llog.c:412:llog_init_handle()) kmalloced '(llh)': 8192 at 000000009ed390fc. 00000100:00000010:2.0:1773941371.041302:0:6296:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 0000000099638eb0. 02000000:00000010:2.0:1773941371.041306:0:6296:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 00000000866b11c7. 00000100:00000010:2.0:1773941371.041311:0:6296:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 0000000038cda836. 02000000:00000010:2.0:1773941371.041331:0:6296:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 16384 at 000000008489a850. 00000400:00000200:2.0:1773941371.041337:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941371.041341:0:6296:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 0000000062778c78. 00000400:00000010:2.0:1773941371.041344:0:6296:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 184 at 0000000058223a8e. 00000100:00000200:2.0:1773941371.041347:0:6296:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 16384 bytes, xid 1860112291464832, portal 25 00000400:00000010:2.0:1773941371.041351:0:6296:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000360aa097. 00000100:00000200:2.0:1773941371.041353:0:6296:0:(niobuf.c:94:ptl_send_buf()) Sending 272 bytes to portal 26, xid 1860112291464832, offset 0 00000400:00000200:2.0:1773941371.041358:0:6296:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941371.041362:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941371.041368:0:6296:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941371.041371:0:6296:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941371.041374:0:6296:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941371.041376:0:6296:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941371.041378:0:6296:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:2.0:1773941371.041384:0:6296:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [17:17:256:4294967295] 192.168.204.114@tcp LPNI seq info [17:17:8:4294967295] 00000400:00000200:2.0:1773941371.041388:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941371.041398:0:6296:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:2.0:1773941371.041415:0:6296:0:(socklnd_cb.c:1001:ksocknal_send()) sending 272 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:2.0:1773941371.041420:0:6296:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 00000000729f4b1f. 00000800:00000200:2.0:1773941371.041426:0:6296:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006663d74c] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:2.0:1773941371.041434:0:6296:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:2.0:1773941371.041438:0:6296:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 00000000729f4b1f type 1, nob 368 niov 1 nkiov 1 00000400:00000200:2.0:1773941371.041458:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:0.0:1773941371.041515:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941371.041519:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 00000000729f4b1f. 00000400:00000200:0.0:1773941371.041523:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941371.041530:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941371.041534:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000360aa097 00000400:00000010:0.0:1773941371.041535:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000360aa097. 00000100:00000200:0.0:1773941371.041541:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@0000000099638eb0 x1860112291464832/t0(0) o503->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8416 e 0 to 0 dl 1773941387 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1773941371.042424:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.042428:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941371.042432:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.042434:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.042436:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.042438:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 8416 into portal 25 MB=0x69bc324600680 00000400:00000200:1.0:1773941371.042441:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.042446:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 8416/8416 into md 0x2b9 [4] + 224 00000400:00000200:1.0:1773941371.042471:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941371.042474:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:1.0:1773941371.042479:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@0000000099638eb0 x1860112291464832/t0(0) o503->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8416 e 0 to 0 dl 1773941387 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1773941371.042525:0:6296:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 0000000062778c78. 00000400:00000200:3.0:1773941371.042534:0:6296:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 0000000058223a8e 00000400:00000010:3.0:1773941371.042537:0:6296:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 184 at 0000000058223a8e. 00000100:00000200:3.0:1773941371.042550:0:6296:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@0000000099638eb0 x1860112291464832/t0(0) o503->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8416 e 0 to 0 dl 1773941387 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941371.042563:0:6296:0:(events.c:102:reply_in_callback()) @@@ unlink req@0000000099638eb0 x1860112291464832/t0(0) o503->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8416 e 0 to 0 dl 1773941387 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1773941371.042580:0:6296:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at 0000000038cda836. 02000000:00000010:3.0:1773941371.042591:0:6296:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 16384 at 000000008489a850. 02000000:00000010:3.0:1773941371.042604:0:6296:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 00000000866b11c7. 00000100:00000010:3.0:1773941371.042609:0:6296:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 0000000099638eb0. 00000040:00000010:3.0:1773941371.042617:0:6296:0:(llog.c:945:llog_process_or_fork()) kmalloced '(lpi)': 96 at 000000009d1a1c6d. 00000020:00000010:1.0:1773941371.042767:0:6320:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at 000000008b6f0a19. 00000020:00000010:1.0:1773941371.042774:0:6320:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at 00000000479775a9. 00000040:00000010:1.0:1773941371.042778:0:6320:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at 00000000aefdd398. 00000020:00000010:1.0:1773941371.042783:0:6320:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at 0000000069c1821b. 00000020:00000010:1.0:1773941371.042789:0:6320:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at 00000000760c755a. 00000020:00000010:1.0:1773941371.042793:0:6320:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at 00000000ebbc95b6. 80000000:00000010:1.0:1773941371.042797:0:6320:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at 00000000998dc707. 40000000:00000010:1.0:1773941371.042801:0:6320:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at 00000000774c2b35. 00100000:00000010:1.0:1773941371.042816:0:6320:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at 00000000e914a1ca. 00040000:00000010:1.0:1773941371.042822:0:6320:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at 00000000adbd3943. 00040000:00000010:1.0:1773941371.042827:0:6320:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at 00000000be2c8c62. 00040000:00000010:1.0:1773941371.042831:0:6320:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at 00000000905556b5. 00000004:00000010:1.0:1773941371.042837:0:6320:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at 000000008f241d19. 00000004:00000010:1.0:1773941371.042845:0:6320:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at 0000000049e32710. 00000004:00000010:1.0:1773941371.042851:0:6320:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at 00000000b552bc66. 00002000:00000010:1.0:1773941371.042857:0:6320:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at 0000000023617eed. 00000004:00000010:1.0:1773941371.042861:0:6320:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at 000000000cff2190. 00000020:00000010:1.0:1773941371.042866:0:6320:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at 000000008978e57b. 00000040:00000010:1.0:1773941371.042876:0:6320:0:(llog.c:570:llog_process_thread()) kmalloced '(buf)': 8192 at 00000000a35a5129. 00000100:00000010:1.0:1773941371.042883:0:6320:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at 00000000859dd066. 02000000:00000010:1.0:1773941371.042895:0:6320:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at 00000000df04275e. 00000100:00000010:1.0:1773941371.042906:0:6320:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at 000000006c43a5cc. 02000000:00000010:1.0:1773941371.042931:0:6320:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 16384 at 00000000627fa483. 00000400:00000200:1.0:1773941371.042936:0:6320:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941371.042940:0:6320:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at 000000009ad73207. 00000400:00000010:1.0:1773941371.042950:0:6320:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 184 at 00000000c4bcb71f. 00000100:00000200:1.0:1773941371.042955:0:6320:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 16384 bytes, xid 1860112291464960, portal 25 00000400:00000010:1.0:1773941371.043023:0:6320:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at 00000000b0aed1e9. 00000100:00000200:1.0:1773941371.043027:0:6320:0:(niobuf.c:94:ptl_send_buf()) Sending 272 bytes to portal 26, xid 1860112291464960, offset 0 00000400:00000200:1.0:1773941371.043038:0:6320:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941371.043044:0:6320:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.043051:0:6320:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941371.043055:0:6320:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941371.043062:0:6320:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941371.043067:0:6320:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941371.043073:0:6320:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:1.0:1773941371.043079:0:6320:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [18:18:256:4294967295] 192.168.204.114@tcp LPNI seq info [18:18:8:4294967295] 00000400:00000200:1.0:1773941371.043085:0:6320:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.043099:0:6320:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:1.0:1773941371.043108:0:6320:0:(socklnd_cb.c:1001:ksocknal_send()) sending 272 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:1.0:1773941371.043123:0:6320:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at 0000000064603a4c. 00000800:00000200:1.0:1773941371.043129:0:6320:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [000000006663d74c] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:1.0:1773941371.043135:0:6320:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:1.0:1773941371.043138:0:6320:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 0000000064603a4c type 1, nob 368 niov 1 nkiov 1 00000400:00000200:1.0:1773941371.043156:0:6320:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:1.0:1773941371.043206:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941371.043209:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at 0000000064603a4c. 00000400:00000200:1.0:1773941371.043214:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941371.043219:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941371.043222:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000b0aed1e9 00000400:00000010:1.0:1773941371.043223:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at 00000000b0aed1e9. 00000100:00000200:1.0:1773941371.043229:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@00000000859dd066 x1860112291464960/t0(0) o502->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941387 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1773941371.044022:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.044026:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941371.044030:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.044033:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.044036:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.044039:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 8472 into portal 25 MB=0x69bc324600700 00000400:00000200:1.0:1773941371.044042:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.044045:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 8472/8472 into md 0x2c1 [4] + 224 00000400:00000200:1.0:1773941371.044058:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941371.044061:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:1.0:1773941371.044065:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@00000000859dd066 x1860112291464960/t0(0) o502->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941387 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1773941371.044105:0:6320:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at 000000009ad73207. 00000400:00000200:1.0:1773941371.044108:0:6320:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md 00000000c4bcb71f 00000400:00000010:1.0:1773941371.044109:0:6320:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 184 at 00000000c4bcb71f. 00000100:00000200:1.0:1773941371.044113:0:6320:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@00000000859dd066 x1860112291464960/t0(0) o502->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941387 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941371.044119:0:6320:0:(events.c:102:reply_in_callback()) @@@ unlink req@00000000859dd066 x1860112291464960/t0(0) o502->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941387 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1773941371.044148:0:6320:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at 000000006c43a5cc. 02000000:00000010:1.0:1773941371.044154:0:6320:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 16384 at 00000000627fa483. 02000000:00000010:1.0:1773941371.044163:0:6320:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at 00000000df04275e. 00000100:00000010:1.0:1773941371.044166:0:6320:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at 00000000859dd066. 00000020:00000010:1.0:1773941371.044180:0:6320:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at 0000000064603a4c. 00000020:00000010:1.0:1773941371.044184:0:6320:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at 0000000064603a4c. 00000020:00000010:1.0:1773941371.044189:0:6320:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 28 at 0000000096fc453f. 00000020:00000010:1.0:1773941371.044194:0:6320:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 112 at 00000000ff68aeb1. 00000020:00000010:1.0:1773941371.044222:0:6320:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 112 at ffff988d0716b700. 00000020:00000010:1.0:1773941371.044227:0:6320:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 28 at ffff988d10648f00. 00000100:00000010:1.0:1773941371.044232:0:6320:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c0700. 02000000:00000010:1.0:1773941371.044237:0:6320:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d0832c000. 00000100:00000010:1.0:1773941371.044244:0:6320:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d05447180. 02000000:00000010:1.0:1773941371.044257:0:6320:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 16384 at ffff988d10838000. 00000400:00000200:1.0:1773941371.044262:0:6320:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941371.044264:0:6320:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d877e8. 00000400:00000010:1.0:1773941371.044267:0:6320:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 184 at ffff988d05447e40. 00000100:00000200:1.0:1773941371.044270:0:6320:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 16384 bytes, xid 1860112291465088, portal 25 00000400:00000010:1.0:1773941371.044273:0:6320:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de990. 00000100:00000200:1.0:1773941371.044275:0:6320:0:(niobuf.c:94:ptl_send_buf()) Sending 272 bytes to portal 26, xid 1860112291465088, offset 0 00000400:00000200:1.0:1773941371.044279:0:6320:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941371.044284:0:6320:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.044289:0:6320:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941371.044293:0:6320:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941371.044296:0:6320:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941371.044297:0:6320:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941371.044299:0:6320:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:1.0:1773941371.044301:0:6320:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [19:19:256:4294967295] 192.168.204.114@tcp LPNI seq info [19:19:8:4294967295] 00000400:00000200:1.0:1773941371.044304:0:6320:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.044312:0:6320:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:1.0:1773941371.044322:0:6320:0:(socklnd_cb.c:1001:ksocknal_send()) sending 272 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:1.0:1773941371.044325:0:6320:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cf9b00. 00000800:00000200:1.0:1773941371.044329:0:6320:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:1.0:1773941371.044333:0:6320:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:1.0:1773941371.044336:0:6320:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cf9b00 type 1, nob 368 niov 1 nkiov 1 00000400:00000200:1.0:1773941371.044344:0:6320:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:1.0:1773941371.044395:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941371.044397:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cf9b00. 00000400:00000200:1.0:1773941371.044402:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941371.044407:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941371.044411:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de990 00000400:00000010:1.0:1773941371.044413:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de990. 00000100:00000200:1.0:1773941371.044420:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c0700 x1860112291465088/t0(0) o502->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941387 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1773941371.045082:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.045087:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941371.045093:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.045096:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.045099:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.045101:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 8472 into portal 25 MB=0x69bc324600780 00000400:00000200:1.0:1773941371.045104:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.045107:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 8472/8472 into md 0x2d1 [4] + 224 00000400:00000200:1.0:1773941371.045120:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941371.045123:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:1.0:1773941371.045128:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c0700 x1860112291465088/t0(0) o502->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941387 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1773941371.045149:0:6320:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d877e8. 00000400:00000200:1.0:1773941371.045152:0:6320:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d05447e40 00000400:00000010:1.0:1773941371.045154:0:6320:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 184 at ffff988d05447e40. 00000100:00000200:1.0:1773941371.045157:0:6320:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c0700 x1860112291465088/t0(0) o502->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941387 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941371.045173:0:6320:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c0700 x1860112291465088/t0(0) o502->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941387 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1773941371.045186:0:6320:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d05447180. 02000000:00000010:1.0:1773941371.045192:0:6320:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 16384 at ffff988d10838000. 02000000:00000010:1.0:1773941371.045200:0:6320:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d0832c000. 00000100:00000010:1.0:1773941371.045203:0:6320:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c0700. 00000020:00000010:1.0:1773941371.045213:0:6320:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at ffff988d02cf9b00. 00000020:00000010:1.0:1773941371.045216:0:6320:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at ffff988d02cf9b00. 00000040:00000010:1.0:1773941371.045218:0:6320:0:(llog.c:891:llog_process_thread()) kvfree 'buf': 8192 at ffff988d107fc000. 00000020:00000010:1.0:1773941371.045227:0:6320:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff988d04d679c0. 00000020:00000010:1.0:1773941371.045239:0:6320:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d0832de00. 00000040:00000010:1.0:1773941371.045243:0:6320:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d0832dc00. 00000020:00000010:1.0:1773941371.045247:0:6320:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d08f0d800. 00000020:00000010:1.0:1773941371.045253:0:6320:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff988d181f1e30. 00000020:00000010:1.0:1773941371.045264:0:6320:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d0832ce00. 80000000:00000010:1.0:1773941371.045266:0:6320:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d209386c0. 40000000:00000010:1.0:1773941371.045270:0:6320:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff988d04d67400. 00100000:00000010:1.0:1773941371.045278:0:6320:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff988d08f08800. 00040000:00000010:1.0:1773941371.045283:0:6320:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d02cf9e00. 00040000:00000010:1.0:1773941371.045288:0:6320:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff988d08f0c800. 00040000:00000010:1.0:1773941371.045294:0:6320:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d0832c200. 00000004:00000010:1.0:1773941371.045298:0:6320:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff988d08f0b000. 00000004:00000010:1.0:1773941371.045303:0:6320:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff988d07205000. 00000004:00000010:1.0:1773941371.045307:0:6320:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff988d08f08000. 00002000:00000010:1.0:1773941371.045310:0:6320:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff988d1081d000. 00000004:00000010:1.0:1773941371.045313:0:6320:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff988d1081fc00. 00000020:00000010:1.0:1773941371.045317:0:6320:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0832da00. 00000040:00000010:1.0:1773941371.045391:0:6296:0:(llog.c:986:llog_process_or_fork()) kfreed 'lpi': 96 at ffff988d18350840. 00000040:00000010:1.0:1773941371.045397:0:6296:0:(llog.c:70:llog_free_handle()) kvfree 'loghandle->lgh_hdr': 8192 at ffff988d0650c000. 00000040:00000010:1.0:1773941371.045401:0:6296:0:(llog.c:72:llog_free_handle()) kfreed 'loghandle': 352 at ffff988d1a238400. 00000020:00000010:1.0:1773941371.045409:0:6296:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d1a246c00. 00000040:00000010:1.0:1773941371.045413:0:6296:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d1a246000. 00000020:00000010:1.0:1773941371.045428:0:6296:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d0835e800. 00000020:00000010:1.0:1773941371.045432:0:6296:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d1a246a00. 80000000:00000010:1.0:1773941371.045437:0:6296:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d183501e0. 00040000:00000010:1.0:1773941371.045441:0:6296:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d1a246200. 20000000:00000010:1.0:1773941371.045448:0:6296:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d1a239e00. 00000020:00000010:1.0:1773941371.045455:0:6296:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d1a247800. 10000000:00000010:1.0:1773941371.045459:0:6296:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d072758c0. 00000020:00000010:1.0:1773941371.045484:0:6296:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d1961ea00. 00000040:00000010:1.0:1773941371.045489:0:6296:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d1961e200. 00000020:00000010:1.0:1773941371.045494:0:6296:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d21759000. 00000020:00000010:1.0:1773941371.045502:0:6296:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d20904a00. 00040000:00000010:1.0:1773941371.045505:0:6296:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d02cf9800. 00040000:00000010:1.0:1773941371.045509:0:6296:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d20905c00. 00000020:00000010:1.0:1773941371.045512:0:6296:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0838b800. 00000020:00000010:1.0:1773941371.045517:0:6296:0:(obd_mount.c:82:lustre_process_log()) kfreed 'lcfg': 184 at ffff988d21c4cb40. 00000020:00000010:1.0:1773941371.045521:0:6296:0:(obd_mount.c:84:lustre_process_log()) kfreed 'bufs': 104 at ffff988d0716bc80. 00000080:00000010:1.0:1773941371.045534:0:6296:0:(llite_lib.c:1502:ll_fill_super()) kmalloced '(dt)': 33 at ffff988d04d67400. 00000080:00000010:1.0:1773941371.045539:0:6296:0:(llite_lib.c:1508:ll_fill_super()) kmalloced '(md)': 33 at ffff988d04d672c0. 00000080:00000010:1.0:1773941371.045548:0:6296:0:(llite_lib.c:348:client_common_fill_super()) kmalloced '(data)': 192 at ffff988d05447180. 00000080:00000010:1.0:1773941371.045552:0:6296:0:(llite_lib.c:352:client_common_fill_super()) kmalloced '(osfs)': 144 at ffff988d05447e40. 00000020:00000010:1.0:1773941371.045563:0:6296:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff988d08f08000. 00000020:00000010:1.0:1773941371.045616:0:6296:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff988d08f0b000. 00000400:00000200:1.0:1773941371.045648:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.045653:0:6296:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000010:1.0:1773941371.045671:0:6296:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c0700. 02000000:00000010:1.0:1773941371.045679:0:6296:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d1081fc00. 40000000:00000010:1.0:1773941371.045717:0:6296:0:(fid_request.c:432:client_fid_init()) kmalloced '(cli->cl_seq)': 224 at ffff988d02cf9800. 40000000:00000010:1.0:1773941371.045720:0:6296:0:(fid_request.c:436:client_fid_init()) kmalloced '(prefix)': 133 at ffff988d05447a80. 40000000:00000010:1.0:1773941371.045862:0:6296:0:(fid_request.c:444:client_fid_init()) kfreed 'prefix': 133 at ffff988d05447a80. 80000000:00000010:1.0:1773941371.045871:0:6296:0:(fld_request.c:150:fld_client_add_target()) kmalloced '(target)': 40 at ffff988d04d67e40. 00020000:00000010:1.0:1773941371.045881:0:6296:0:(lu_tgt_descs.c:102:lu_qos_add_tgt()) kmalloced '(svr)': 104 at ffff988d0716bc80. 00000100:00000010:1.0:1773941371.045889:0:6296:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c3100. 02000000:00000010:1.0:1773941371.045896:0:6296:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff988d02cf9e00. 00000020:00000010:0.0:1773941371.045932:0:2402:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff988d054ecb78. 00000020:00000010:0.0:1773941371.045938:0:2402:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d065c4000. 80000000:00000010:0.0:1773941371.045942:0:2402:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d06580cc0. 00000020:00000010:1.0:1773941371.045944:0:6296:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff988d08f0c800. 40000000:00000010:0.0:1773941371.045946:0:2402:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff988d10872500. 00000008:00000010:0.0:1773941371.045954:0:2402:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff988d203ece18. 00000400:00000200:1.0:1773941371.045991:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00020000:00000010:0.0:1773941371.045996:0:2402:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff988d0a222900. 00000400:00000200:1.0:1773941371.046001:0:6296:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00008000:00000010:0.0:1773941371.046005:0:2402:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d033366d8. 00100000:00000010:0.0:1773941371.046010:0:2402:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff988d20337800. 00000100:00000010:1.0:1773941371.046012:0:6296:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c0380. 00040000:00000010:0.0:1773941371.046016:0:2402:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d113e9200. 02000000:00000010:1.0:1773941371.046022:0:6296:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d1081d000. 00040000:00000010:0.0:1773941371.046022:0:2402:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff988d20337000. 00040000:00000010:0.0:1773941371.046029:0:2402:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d065c5200. 00000004:00000010:0.0:1773941371.046034:0:2402:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff988d20334000. 00000004:00000010:0.0:1773941371.046039:0:2402:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff988d065f4000. 40000000:00000010:1.0:1773941371.046040:0:6296:0:(fid_request.c:432:client_fid_init()) kmalloced '(cli->cl_seq)': 224 at ffff988d02cf9b00. 00000004:00000010:0.0:1773941371.046044:0:2402:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff988d20333000. 40000000:00000010:1.0:1773941371.046045:0:6296:0:(fid_request.c:436:client_fid_init()) kmalloced '(prefix)': 133 at ffff988d05447a80. 00002000:00000010:0.0:1773941371.046052:0:2402:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff988d065fac00. 00000004:00000010:0.0:1773941371.046064:0:2402:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff988d065f9800. 00000080:00000010:0.0:1773941371.046072:0:2402:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff988d065730a8. 00000080:00000010:0.0:1773941371.046077:0:2402:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff988d089cb908. 00000008:00000010:0.0:1773941371.046083:0:2402:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff988d107c22c0. 00020000:00000010:0.0:1773941371.046088:0:2402:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff988d18154fc0. 00008000:00000010:0.0:1773941371.046093:0:2402:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff988d065233e8. 00000080:00000010:0.0:1773941371.046098:0:2402:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff988d07fdbe10. 00000020:00000010:0.0:1773941371.046122:0:2401:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff988d054ed080. 00000020:00000010:0.0:1773941371.046127:0:2401:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d065c5600. 40000000:00000010:1.0:1773941371.046134:0:6296:0:(fid_request.c:444:client_fid_init()) kfreed 'prefix': 133 at ffff988d05447a80. 80000000:00000010:0.0:1773941371.046134:0:2401:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d06580ae0. 40000000:00000010:0.0:1773941371.046142:0:2401:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff988d108723c0. 80000000:00000010:1.0:1773941371.046145:0:6296:0:(fld_request.c:150:fld_client_add_target()) kmalloced '(target)': 40 at ffff988d1a2144c0. 00000008:00000010:0.0:1773941371.046149:0:2401:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff988d203e8b28. 00000100:00000010:1.0:1773941371.046156:0:6296:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c1c00. 00020000:00000010:0.0:1773941371.046157:0:2401:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff988d0a222e40. 00008000:00000010:0.0:1773941371.046164:0:2401:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d03337170. 02000000:00000010:1.0:1773941371.046166:0:6296:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff988d02cf9400. 00100000:00000010:0.0:1773941371.046182:0:2401:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff988d111b7800. 00040000:00000010:0.0:1773941371.046189:0:2401:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d113e9f00. 00040000:00000010:0.0:1773941371.046193:0:2401:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff988d111b7000. 00040000:00000010:0.0:1773941371.046197:0:2401:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d065c4400. 00800000:00000010:1.0:1773941371.046203:0:6296:0:(lmv_obd.c:1417:lmv_statfs()) kmalloced '(temp)': 144 at ffff988d05447a80. 00000004:00000010:0.0:1773941371.046203:0:2401:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff988d111b4000. 00000100:00000010:1.0:1773941371.046213:0:6296:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c0a80. 00000004:00000010:0.0:1773941371.046214:0:2401:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff988d065f7000. 02000000:00000010:1.0:1773941371.046221:0:6296:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff988d02cf9a00. 00000004:00000010:0.0:1773941371.046221:0:2401:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff988d111b3000. 00002000:00000010:0.0:1773941371.046226:0:2401:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff988d065f9000. 00000100:00000010:1.0:1773941371.046229:0:6296:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d05447900. 00000004:00000010:0.0:1773941371.046231:0:2401:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff988d065fbc00. 00000080:00000010:0.0:1773941371.046238:0:2401:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff988d06573498. 00000080:00000010:0.0:1773941371.046243:0:2401:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff988d089c9240. 00000008:00000010:0.0:1773941371.046248:0:2401:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff988d107c34a0. 00020000:00000010:0.0:1773941371.046255:0:2401:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff988d18155680. 00008000:00000010:0.0:1773941371.046262:0:2401:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff988d06523388. 02000000:00000010:1.0:1773941371.046267:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d1081c800. 00000080:00000010:0.0:1773941371.046268:0:2401:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff988d07fdbc30. 00000400:00000200:1.0:1773941371.046278:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941371.046281:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d877e8. 00000400:00000010:1.0:1773941371.046288:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de990. 00000100:00000200:1.0:1773941371.046293:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291465216, portal 10 00000020:00000010:1.0:1773941371.046301:0:2400:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffff988d107fc000. 00000400:00000010:1.0:1773941371.046308:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113dee58. 00000100:00000200:1.0:1773941371.046312:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1860112291465216, offset 0 00000400:00000200:1.0:1773941371.046321:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941371.046327:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.046333:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941371.046337:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941371.046342:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941371.046346:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941371.046351:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:1.0:1773941371.046354:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [20:20:256:4294967295] 192.168.204.114@tcp LPNI seq info [20:20:8:4294967295] 00000400:00000200:1.0:1773941371.046359:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.046377:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:1.0:1773941371.046382:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:1.0:1773941371.046387:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cf9900. 00000800:00000200:1.0:1773941371.046392:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:1.0:1773941371.046398:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:1.0:1773941371.046400:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cf9900 type 1, nob 616 niov 1 nkiov 1 00000400:00000200:1.0:1773941371.046412:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 02000000:00000010:1.0:1773941371.046429:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d1081d800. 00000400:00000200:1.0:1773941371.046431:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941371.046439:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d87840. 00000400:00000010:1.0:1773941371.046447:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de6e8. 00000100:00000200:1.0:1773941371.046450:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291465472, portal 10 00000020:00000010:1.0:1773941371.046456:0:2400:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffff988d107f8000. 00000400:00000010:1.0:1773941371.046461:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113debb0. 00000100:00000200:1.0:1773941371.046466:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1860112291465472, offset 0 00000400:00000200:1.0:1773941371.046471:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941371.046474:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.046476:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941371.046478:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941371.046479:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941371.046481:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941371.046482:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:1.0:1773941371.046484:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [21:21:255:4294967295] 192.168.204.114@tcp LPNI seq info [21:21:7:4294967295] 00000400:00000200:1.0:1773941371.046486:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.046490:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:1.0:1773941371.046493:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:1.0:1773941371.046496:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cf9500. 00000800:00000200:1.0:1773941371.046498:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:1.0:1773941371.046501:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:1.0:1773941371.046503:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cf9500 type 1, nob 616 niov 1 nkiov 1 00000400:00000200:1.0:1773941371.046513:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:1.0:1773941371.046554:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941371.046556:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cf9900. 00000400:00000200:1.0:1773941371.046559:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941371.046562:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941371.046566:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113dee58 00000400:00000010:1.0:1773941371.046568:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113dee58. 00000100:00000200:1.0:1773941371.046576:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c0700 x1860112291465216/t0(0) o38->lustre-MDT0000-mdc-ffff988d07f1c800@192.168.204.114@tcp:12/10 lens 520/544 e 0 to 0 dl 1773941381 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1773941371.046628:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941371.046631:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cf9500. 00000400:00000200:1.0:1773941371.046646:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941371.046650:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941371.046652:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113debb0 00000400:00000010:1.0:1773941371.046654:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113debb0. 00000100:00000200:1.0:1773941371.046660:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c0380 x1860112291465472/t0(0) o38->lustre-MDT0001-mdc-ffff988d07f1c800@192.168.204.114@tcp:12/10 lens 520/544 e 0 to 0 dl 1773941381 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1773941371.047943:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.047948:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941371.047954:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.047958:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.048015:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.048018:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 10 MB=0x69bc324600800 00000400:00000200:1.0:1773941371.048023:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.048026:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.114@tcp of length 416/416 into md 0x2e1 [1] + 0 00000400:00000200:1.0:1773941371.048036:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941371.048040:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:1.0:1773941371.048047:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c0700 x1860112291465216/t0(0) o38->lustre-MDT0000-mdc-ffff988d07f1c800@192.168.204.114@tcp:12/10 lens 520/544 e 0 to 0 dl 1773941381 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1773941371.048070:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.048074:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941371.048083:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.048085:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.048087:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.048088:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 10 MB=0x69bc324600900 00000400:00000200:1.0:1773941371.048090:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941371.048092:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.114@tcp of length 416/416 into md 0x2f1 [1] + 0 00000400:00000200:1.0:1773941371.048098:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941371.048100:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:1.0:1773941371.048104:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c0380 x1860112291465472/t0(0) o38->lustre-MDT0001-mdc-ffff988d07f1c800@192.168.204.114@tcp:12/10 lens 520/544 e 0 to 0 dl 1773941381 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1773941371.048121:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d877e8. 00000400:00000200:1.0:1773941371.048123:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de990 00000400:00000010:1.0:1773941371.048124:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de990. 00000100:00000200:1.0:1773941371.048128:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c0700 x1860112291465216/t0(0) o38->lustre-MDT0000-mdc-ffff988d07f1c800@192.168.204.114@tcp:12/10 lens 520/544 e 0 to 0 dl 1773941381 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941371.048133:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c0700 x1860112291465216/t0(0) o38->lustre-MDT0000-mdc-ffff988d07f1c800@192.168.204.114@tcp:12/10 lens 520/544 e 0 to 0 dl 1773941381 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:0.0:1773941371.050041:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d18350d20. 00000020:00000010:3.0:1773941371.051386:0:6319:0:(lproc_ptlrpc.c:1375:ldebugfs_import_seq_write()) kmalloced '(kbuf)': 34 at ffff988d07275140. 00000020:00000010:3.0:1773941371.051397:0:6319:0:(lproc_ptlrpc.c:1427:ldebugfs_import_seq_write()) kfreed 'kbuf': 34 at ffff988d07275140. 00000020:00000010:2.0:1773941371.053334:0:6322:0:(lproc_ptlrpc.c:1375:ldebugfs_import_seq_write()) kmalloced '(kbuf)': 34 at ffff988d07176600. 00000100:00000200:2.0:1773941371.053357:0:6322:0:(events.c:560:ptlrpc_uuid_to_peer()) 192.168.204.114@tcp->12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941371.053365:0:6322:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941371.053371:0:6322:0:(peer.c:1471:LNetPrimaryNID()) NID 192.168.204.114@tcp primary NID 192.168.204.114@tcp rc 0 00000020:00000010:2.0:1773941371.053376:0:6322:0:(lproc_ptlrpc.c:1427:ldebugfs_import_seq_write()) kfreed 'kbuf': 34 at ffff988d07176600. 00000100:00000010:1.0:1773941371.058114:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580c60. 00000020:00000010:2.0:1773941371.060598:0:6325:0:(lproc_ptlrpc.c:1375:ldebugfs_import_seq_write()) kmalloced '(kbuf)': 34 at ffff988d07176340. 00000020:00000010:2.0:1773941371.060608:0:6325:0:(lproc_ptlrpc.c:1427:ldebugfs_import_seq_write()) kfreed 'kbuf': 34 at ffff988d07176340. 00000020:00000010:0.0:1773941371.062066:0:6324:0:(lproc_ptlrpc.c:1375:ldebugfs_import_seq_write()) kmalloced '(kbuf)': 34 at ffff988d108727c0. 00000020:00000010:0.0:1773941371.062076:0:6324:0:(lproc_ptlrpc.c:1427:ldebugfs_import_seq_write()) kfreed 'kbuf': 34 at ffff988d108727c0. 00000020:00000010:0.0:1773941372.027037:0:41:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 448 at ffff988d065c4c00. 00000400:00000010:2.0:1773941372.048398:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d87840. 00000400:00000200:2.0:1773941372.048408:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de6e8 00000400:00000010:2.0:1773941372.048410:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de6e8. 00000100:00000200:2.0:1773941372.048423:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c0380 x1860112291465472/t0(0) o38->lustre-MDT0001-mdc-ffff988d07f1c800@192.168.204.114@tcp:12/10 lens 520/544 e 0 to 0 dl 1773941381 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941372.048435:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c0380 x1860112291465472/t0(0) o38->lustre-MDT0001-mdc-ffff988d07f1c800@192.168.204.114@tcp:12/10 lens 520/544 e 0 to 0 dl 1773941381 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1773941372.048437:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d065fa000. 02000000:00000010:3.0:1773941372.048447:0:6296:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d02ce4400. 00000400:00000200:0.0:1773941372.048450:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000020:00000010:2.0:1773941372.048452:0:2400:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffff988d06508000. 00000400:00000010:0.0:1773941372.048457:0:2402:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f5d8. 00000400:00000200:3.0:1773941372.048463:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941372.048465:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833d48. 00000400:00000010:3.0:1773941372.048468:0:6296:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccdb58. 00000100:00000200:0.0:1773941372.048474:0:2402:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291465344, portal 10 00000400:00000010:3.0:1773941372.048475:0:6296:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac880. 00000100:00000200:3.0:1773941372.048483:0:6296:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291465728, portal 10 00000020:00000010:0.0:1773941372.048486:0:2402:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffff988d111fa000. 00000020:00000010:3.0:1773941372.048491:0:6296:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffff988d0337a000. 00000400:00000010:0.0:1773941372.048495:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833198. 00000400:00000010:3.0:1773941372.048498:0:6296:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac550. 00000100:00000200:0.0:1773941372.048502:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1860112291465344, offset 0 00000100:00000200:3.0:1773941372.048504:0:6296:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1860112291465728, offset 0 00000400:00000200:3.0:1773941372.048518:0:6296:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941372.048519:0:2402:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941372.048532:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941372.048533:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941372.048545:0:6296:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941372.048552:0:6296:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941372.048555:0:2402:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941372.048560:0:6296:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941372.048567:0:6296:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941372.048567:0:2402:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941372.048574:0:6296:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:0.0:1773941372.048577:0:2402:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941372.048583:0:6296:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [22:22:256:4294967295] 192.168.204.114@tcp LPNI seq info [22:22:8:4294967295] 00000400:00000200:0.0:1773941372.048585:0:2402:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 02000000:00000010:1.0:1773941372.048591:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d1081c800. 00000400:00000200:0.0:1773941372.048594:0:2402:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941372.048597:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 02000000:00000010:1.0:1773941372.048600:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d1081fc00. 00000400:00000200:0.0:1773941372.048604:0:2402:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [23:23:256:4294967295] 192.168.204.114@tcp LPNI seq info [23:23:8:4294967295] 00000100:00000010:1.0:1773941372.048606:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c0700. 02000000:00000010:1.0:1773941372.048613:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d1081d800. 00000400:00000200:0.0:1773941372.048629:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 02000000:00000010:1.0:1773941372.048630:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d1081d000. 00000100:00000010:1.0:1773941372.048635:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c0380. 00000400:00000200:0.0:1773941372.048649:0:2402:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:0.0:1773941372.048658:0:2402:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941372.048666:0:6296:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000010:0.0:1773941372.048667:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d113e9c00. 00000800:00000200:3.0:1773941372.048677:0:6296:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000200:0.0:1773941372.048682:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000010:3.0:1773941372.048685:0:6296:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d09a9c100. 00000800:00000200:3.0:1773941372.048692:0:6296:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:0.0:1773941372.048697:0:2402:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:3.0:1773941372.048702:0:6296:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:0.0:1773941372.048705:0:2402:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d113e9c00 type 1, nob 320 niov 1 nkiov 1 00000800:00000200:3.0:1773941372.048707:0:6296:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d09a9c100 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1773941372.048730:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941372.048731:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000010:0.0:1773941372.048742:0:2402:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580300. 00000100:00000010:0.0:1773941372.048757:0:2402:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580000. 00000800:00000200:1.0:1773941372.048760:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941372.048763:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d113e9c00. 00000400:00000200:1.0:1773941372.048769:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941372.048775:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941372.048779:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833198 00000400:00000010:1.0:1773941372.048780:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833198. 00000100:00000200:1.0:1773941372.048792:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c3100 x1860112291465344/t0(0) o41->lustre-MDT0000-mdc-ffff988d07f1c800@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941388 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1773941372.048792:0:2401:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d065f9400. 00000400:00000200:0.0:1773941372.048802:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941372.048806:0:2401:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f7e8. 00000400:00000010:0.0:1773941372.048811:0:2401:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833088. 00000100:00000200:0.0:1773941372.048824:0:2401:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291465600, portal 10 00000020:00000010:0.0:1773941372.048832:0:2401:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffff988d111fe000. 00000400:00000010:0.0:1773941372.048838:0:2401:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833990. 00000100:00000200:0.0:1773941372.048841:0:2401:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1860112291465600, offset 0 00000400:00000200:0.0:1773941372.048848:0:2401:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941372.048854:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:1.0:1773941372.048857:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000400:00000200:0.0:1773941372.048860:0:2401:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000800:00000010:1.0:1773941372.048861:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d09a9c100. 00000400:00000200:1.0:1773941372.048866:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941372.048868:0:2401:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941372.048872:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941372.048876:0:2401:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941372.048880:0:2401:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941372.048882:0:2401:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:0.0:1773941372.048885:0:2401:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [24:24:255:4294967295] 192.168.204.114@tcp LPNI seq info [24:24:7:4294967295] 00000400:00000200:0.0:1773941372.048891:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941372.048897:0:2401:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000400:00000200:1.0:1773941372.048903:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac550 00000800:00000200:0.0:1773941372.048905:0:2401:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941372.048907:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac550. 00000800:00000010:0.0:1773941372.048912:0:2401:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d113e9000. 00000100:00000200:1.0:1773941372.048914:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c0a80 x1860112291465728/t0(0) o41->lustre-MDT0000-mdc-ffff988d07f1c800@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941388 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1773941372.048922:0:2401:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:0.0:1773941372.048929:0:2401:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:0.0:1773941372.048933:0:2401:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d113e9000 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:0.0:1773941372.048948:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:1.0:1773941372.048958:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941372.048987:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d113e9000. 00000400:00000200:1.0:1773941372.048990:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941372.048994:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941372.048997:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833990 00000400:00000010:1.0:1773941372.048998:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833990. 00000100:00000200:1.0:1773941372.049002:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c1c00 x1860112291465600/t0(0) o41->lustre-MDT0001-mdc-ffff988d07f1c800@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941388 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1773941372.049995:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941372.049999:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941372.050004:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941372.050007:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941372.050011:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941372.050014:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 368 into portal 10 MB=0x69bc324600a00 00000400:00000200:1.0:1773941372.050017:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941372.050021:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.114@tcp of length 368/368 into md 0x309 [1] + 224 00000400:00000200:1.0:1773941372.050029:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941372.050032:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:1.0:1773941372.050036:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c0a80 x1860112291465728/t0(0) o41->lustre-MDT0000-mdc-ffff988d07f1c800@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941388 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1773941372.050066:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941372.050068:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941372.050070:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941372.050072:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941372.050074:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941372.050077:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 368 into portal 10 MB=0x69bc324600880 00000400:00000200:1.0:1773941372.050080:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941372.050082:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.114@tcp of length 368/368 into md 0x301 [1] + 224 00000400:00000200:1.0:1773941372.050087:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941372.050090:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:1.0:1773941372.050095:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c3100 x1860112291465344/t0(0) o41->lustre-MDT0000-mdc-ffff988d07f1c800@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941388 ref 1 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1773941372.050111:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941372.050113:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941372.050115:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941372.050118:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941372.050124:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941372.050129:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 368 into portal 10 MB=0x69bc324600980 00000400:00000200:1.0:1773941372.050134:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941372.050140:0:6296:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccdb58. 00000400:00000200:3.0:1773941372.050151:0:6296:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac880 00000400:00000010:3.0:1773941372.050153:0:6296:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac880. 00000400:00000200:1.0:1773941372.050159:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.114@tcp of length 368/368 into md 0x319 [1] + 224 00000400:00000010:0.0:1773941372.050166:0:2402:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f5d8. 00000100:00000200:3.0:1773941372.050167:0:6296:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c0a80 x1860112291465728/t0(0) o41->lustre-MDT0000-mdc-ffff988d07f1c800@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941388 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1773941372.050168:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941372.050182:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833d48 00000400:00000010:0.0:1773941372.050192:0:2402:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833d48. 00000400:00000200:1.0:1773941372.050193:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:0.0:1773941372.050213:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c3100 x1860112291465344/t0(0) o41->lustre-MDT0000-mdc-ffff988d07f1c800@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941388 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941372.050217:0:6296:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c0a80 x1860112291465728/t0(0) o41->lustre-MDT0000-mdc-ffff988d07f1c800@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941388 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941372.050222:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c1c00 x1860112291465600/t0(0) o41->lustre-MDT0001-mdc-ffff988d07f1c800@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941388 ref 1 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941372.050289:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c3100 x1860112291465344/t0(0) o41->lustre-MDT0000-mdc-ffff988d07f1c800@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941388 ref 1 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1773941372.050319:0:6296:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d05447900. 02000000:00000010:0.0:1773941372.050324:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d065fa000. 02000000:00000010:3.0:1773941372.050329:0:6296:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d02ce4400. 02000000:00000010:0.0:1773941372.050333:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff988d02cf9e00. 02000000:00000010:3.0:1773941372.050337:0:6296:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff988d02cf9a00. 00000100:00000010:3.0:1773941372.050343:0:6296:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c0a80. 00000100:00000010:0.0:1773941372.050343:0:2402:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c3100. 00800000:00000010:3.0:1773941372.050350:0:6296:0:(lmv_obd.c:1477:lmv_statfs()) kfreed 'temp': 144 at ffff988d05447a80. 00000100:00000010:3.0:1773941372.050360:0:6296:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d111f2a00. 02000000:00000010:3.0:1773941372.050376:0:6296:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d038be000. 00000400:00000010:0.0:1773941372.050376:0:2401:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f7e8. 00000400:00000200:0.0:1773941372.050391:0:2401:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833088 00000400:00000010:0.0:1773941372.050395:0:2401:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833088. 00000100:00000010:3.0:1773941372.050399:0:6296:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1056ce40. 00000100:00000200:0.0:1773941372.050408:0:2401:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c1c00 x1860112291465600/t0(0) o41->lustre-MDT0001-mdc-ffff988d07f1c800@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941388 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1773941372.050416:0:6296:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d1127f800. 00000100:00000200:0.0:1773941372.050424:0:2401:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c1c00 x1860112291465600/t0(0) o41->lustre-MDT0001-mdc-ffff988d07f1c800@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941388 ref 1 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941372.050425:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941372.050429:0:6296:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccdb58. 00000400:00000010:3.0:1773941372.050437:0:6296:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac880. 00000100:00000200:3.0:1773941372.050444:0:6296:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291465856, portal 10 00000400:00000010:3.0:1773941372.050449:0:6296:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac4c8. 00000100:00000200:3.0:1773941372.050452:0:6296:0:(niobuf.c:94:ptl_send_buf()) Sending 440 bytes to portal 12, xid 1860112291465856, offset 0 00000400:00000200:3.0:1773941372.050459:0:6296:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 02000000:00000010:0.0:1773941372.050462:0:2401:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d065f9400. 00000400:00000200:3.0:1773941372.050467:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 02000000:00000010:0.0:1773941372.050472:0:2401:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff988d02cf9400. 00000400:00000200:3.0:1773941372.050476:0:6296:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000100:00000010:0.0:1773941372.050482:0:2401:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c1c00. 00000400:00000200:3.0:1773941372.050483:0:6296:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941372.050491:0:6296:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941372.050495:0:6296:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941372.050498:0:6296:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941372.050501:0:6296:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [25:25:256:4294967295] 192.168.204.114@tcp LPNI seq info [25:25:8:4294967295] 00000400:00000200:3.0:1773941372.050505:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941372.050518:0:6296:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:3.0:1773941372.050523:0:6296:0:(socklnd_cb.c:1001:ksocknal_send()) sending 440 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:3.0:1773941372.050527:0:6296:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d09a9c100. 00000800:00000200:3.0:1773941372.050532:0:6296:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:3.0:1773941372.050538:0:6296:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:3.0:1773941372.050542:0:6296:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d09a9c100 type 1, nob 536 niov 1 nkiov 1 00000400:00000200:3.0:1773941372.050565:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:1.0:1773941372.050625:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941372.050628:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d09a9c100. 00000400:00000200:1.0:1773941372.050631:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941372.050636:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941372.050648:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac4c8 00000400:00000010:1.0:1773941372.050651:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac4c8. 00000100:00000200:1.0:1773941372.050658:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d111f2a00 x1860112291465856/t0(0) o40->lustre-MDT0000-mdc-ffff988d07f1c800@192.168.204.114@tcp:12/10 lens 440/448 e 0 to 0 dl 1773941388 ref 3 fl Rpc:r/209/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1773941372.051184:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941372.051187:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941372.051191:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941372.051193:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941372.051195:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941372.051197:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 448 into portal 10 MB=0x69bc324600a80 00000400:00000200:1.0:1773941372.051200:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941372.051203:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.114@tcp of length 448/448 into md 0x329 [1] + 224 00000400:00000200:1.0:1773941372.051213:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941372.051217:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:1.0:1773941372.051224:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d111f2a00 x1860112291465856/t0(0) o40->lustre-MDT0000-mdc-ffff988d07f1c800@192.168.204.114@tcp:12/10 lens 440/448 e 0 to 0 dl 1773941388 ref 2 fl Rpc:Qr/209/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1773941372.051273:0:6296:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccdb58. 00000400:00000200:3.0:1773941372.051279:0:6296:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac880 00000400:00000010:3.0:1773941372.051280:0:6296:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac880. 00000100:00000200:3.0:1773941372.051287:0:6296:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d111f2a00 x1860112291465856/t0(0) o40->lustre-MDT0000-mdc-ffff988d07f1c800@192.168.204.114@tcp:12/10 lens 440/448 e 0 to 0 dl 1773941388 ref 2 fl Rpc:RQ/209/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941372.051295:0:6296:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d111f2a00 x1860112291465856/t0(0) o40->lustre-MDT0000-mdc-ffff988d07f1c800@192.168.204.114@tcp:12/10 lens 440/448 e 0 to 0 dl 1773941388 ref 2 fl Rpc:RQU/209/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1773941372.051310:0:6296:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1056ce40. 00000002:00000200:3.0:1773941372.051315:0:6296:0:(mdc_request.c:134:mdc_get_root()) root fid=[0x200000007:0x1:0x0], last_committed=0 02000000:00000010:3.0:1773941372.051318:0:6296:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d1127f800. 02000000:00000010:3.0:1773941372.051321:0:6296:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d038be000. 00000100:00000010:3.0:1773941372.051323:0:6296:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d111f2a00. 00000020:00000010:3.0:1773941372.051342:0:6296:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff988d0835d000. 00000020:00000010:3.0:1773941372.051361:0:6296:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff988d08358000. 00000400:00000200:3.0:1773941372.051373:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941372.051378:0:6296:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000010:3.0:1773941372.051392:0:6296:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d111f2a00. 02000000:00000010:3.0:1773941372.051399:0:6296:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d1127f800. 02000000:00000010:1.0:1773941372.051445:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d1081c800. 00000400:00000200:1.0:1773941372.051452:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941372.051455:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d877e8. 00000400:00000010:1.0:1773941372.051465:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de990. 00000020:00000010:3.0:1773941372.051466:0:6296:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff988d08358800. 00000100:00000200:1.0:1773941372.051472:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291465984, portal 4 00000400:00000200:3.0:1773941372.051479:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941372.051484:0:6296:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000010:3.0:1773941372.051491:0:6296:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d111f2300. 00000100:00000010:0.0:1773941372.051491:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10835180. 00000020:00000010:1.0:1773941372.051492:0:2400:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffff988d18a1a000. 02000000:00000010:3.0:1773941372.051511:0:6296:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d02ce4400. 00000400:00000010:1.0:1773941372.051511:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113debb0. 02000000:00000010:0.0:1773941372.051518:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff988d113e9100. 00000100:00000200:1.0:1773941372.051525:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1860112291465984, offset 0 00000400:00000200:1.0:1773941372.051532:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941372.051537:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941372.051542:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941372.051545:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941372.051548:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941372.051552:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941372.051555:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:1.0:1773941372.051562:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [26:26:256:4294967295] 192.168.204.114@tcp LPNI seq info [26:26:8:4294967295] 00000400:00000200:1.0:1773941372.051566:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000080:00000010:3.0:1773941372.051570:0:6296:0:(llite_lib.c:738:client_common_fill_super()) kmalloced '(op_data)': 384 at ffff988d038be000. 00000400:00000200:1.0:1773941372.051579:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:1.0:1773941372.051584:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:1.0:1773941372.051588:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cf9500. 00000800:00000200:1.0:1773941372.051592:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 02000000:00000010:0.0:1773941372.051593:0:2401:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff988d065c5400. 00000020:00000010:3.0:1773941372.051597:0:6296:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 640 at ffff988d2036c000. 00000800:00000200:1.0:1773941372.051602:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000400:00000200:0.0:1773941372.051607:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:1.0:1773941372.051611:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cf9500 type 1, nob 616 niov 1 nkiov 1 00000400:00000010:0.0:1773941372.051626:0:2401:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f7e8. 00000100:00000010:3.0:1773941372.051632:0:6296:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d111f3b80. 00000400:00000010:0.0:1773941372.051636:0:2401:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833088. 00000400:00000200:1.0:1773941372.051641:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 02000000:00000010:3.0:1773941372.051644:0:6296:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d038bee00. 00000100:00000200:0.0:1773941372.051644:0:2401:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112291466240, portal 25 00000400:00000010:0.0:1773941372.051657:0:2401:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833d48. 00000100:00000010:3.0:1773941372.051660:0:6296:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1056ce40. 00000100:00000200:0.0:1773941372.051661:0:2401:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1860112291466240, offset 0 02000000:00000010:3.0:1773941372.051673:0:6296:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff988d038bfe00. 00000400:00000200:0.0:1773941372.051674:0:2401:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941372.051680:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941372.051684:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941372.051694:0:6296:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccdb58. 00000800:00000200:1.0:1773941372.051696:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000400:00000200:0.0:1773941372.051701:0:2401:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000010:3.0:1773941372.051703:0:6296:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac880. 00000800:00000010:1.0:1773941372.051704:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cf9500. 00000100:00000200:3.0:1773941372.051712:0:6296:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112291466368, portal 10 00000400:00000200:0.0:1773941372.051713:0:2401:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941372.051714:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000010:3.0:1773941372.051726:0:6296:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac440. 00000400:00000200:0.0:1773941372.051726:0:2401:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941372.051727:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:3.0:1773941372.051741:0:6296:0:(niobuf.c:94:ptl_send_buf()) Sending 264 bytes to portal 29, xid 1860112291466368, offset 0 00000400:00000200:0.0:1773941372.051742:0:2401:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941372.051754:0:2401:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:0.0:1773941372.051760:0:2401:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [27:27:255:4294967295] 192.168.204.114@tcp LPNI seq info [27:27:7:4294967295] 00000400:00000200:0.0:1773941372.051766:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941372.051774:0:2401:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000400:00000200:1.0:1773941372.051781:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113debb0 00000800:00000200:0.0:1773941372.051781:0:2401:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941372.051787:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113debb0. 00000800:00000010:0.0:1773941372.051790:0:2401:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d113e9300. 00000400:00000200:3.0:1773941372.051798:0:6296:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000100:00000200:1.0:1773941372.051799:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d111f2a00 x1860112291465984/t0(0) o8->lustre-OST0000-osc-ffff988d07f1c800@192.168.204.114@tcp:28/4 lens 520/544 e 0 to 0 dl 1773941382 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941372.051813:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:0.0:1773941372.051818:0:2401:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000400:00000200:3.0:1773941372.051824:0:6296:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941372.051829:0:6296:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000800:00000200:0.0:1773941372.051836:0:2401:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000400:00000200:3.0:1773941372.051837:0:6296:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 02000000:00000010:1.0:1773941372.051840:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d1081e400. 00000400:00000200:3.0:1773941372.051847:0:6296:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000800:00000200:0.0:1773941372.051849:0:2401:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d113e9300 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:1.0:1773941372.051855:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941372.051859:0:6296:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000010:1.0:1773941372.051862:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d87370. 00000400:00000010:1.0:1773941372.051867:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113debb0. 00000400:00000200:3.0:1773941372.051871:0:6296:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [28:28:255:4294967295] 192.168.204.114@tcp LPNI seq info [28:28:7:4294967295] 00000100:00000200:1.0:1773941372.051873:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291466112, portal 4 00000400:00000200:0.0:1773941372.051878:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000020:00000010:1.0:1773941372.051888:0:2400:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffff988d18a1e000. 00000400:00000200:3.0:1773941372.051894:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941372.051894:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113dee58. 00000100:00000200:1.0:1773941372.051902:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1860112291466112, offset 0 00000400:00000200:1.0:1773941372.051912:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941372.051921:0:6296:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000400:00000200:1.0:1773941372.051932:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:3.0:1773941372.051936:0:6296:0:(socklnd_cb.c:1001:ksocknal_send()) sending 264 bytes in 1 frags to 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941372.051941:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000800:00000010:3.0:1773941372.051942:0:6296:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d09a9ca00. 00000400:00000200:1.0:1773941372.051946:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000800:00000200:0.0:1773941372.051950:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000400:00000200:1.0:1773941372.051955:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941372.051996:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000800:00000010:0.0:1773941372.051997:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d113e9300. 00000400:00000200:0.0:1773941372.052004:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941372.052009:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:0.0:1773941372.052014:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941372.052023:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [29:29:254:4294967295] 192.168.204.114@tcp LPNI seq info [29:29:6:4294967295] 00000400:00000200:1.0:1773941372.052027:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:3.0:1773941372.052036:0:6296:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000400:00000200:1.0:1773941372.052038:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000400:00000200:0.0:1773941372.052054:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833d48 00000800:00000200:3.0:1773941372.052055:0:6296:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:1.0:1773941372.052055:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000200:3.0:1773941372.052068:0:6296:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d09a9ca00 type 1, nob 360 niov 1 nkiov 1 00000400:00000010:0.0:1773941372.052068:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833d48. 00000800:00000010:1.0:1773941372.052069:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cf9500. 00000400:00000200:3.0:1773941372.052094:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:1.0:1773941372.052096:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000100:00000200:0.0:1773941372.052097:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10835180 x1860112291466240/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941388 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1773941372.052111:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:1.0:1773941372.052115:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cf9500 type 1, nob 616 niov 1 nkiov 1 00000400:00000200:1.0:1773941372.052123:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:0.0:1773941372.052138:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941372.052142:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d09a9ca00. 00000400:00000200:0.0:1773941372.052146:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941372.052151:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941372.052154:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac440 00000400:00000010:0.0:1773941372.052156:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac440. 00000100:00000200:0.0:1773941372.052162:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d111f3b80 x1860112291466368/t0(0) o900->lustre-MDT0000-mdc-ffff988d07f1c800@192.168.204.114@tcp:29/10 lens 264/248 e 0 to 0 dl 1773941388 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1773941372.052205:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941372.052209:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cf9500. 00000400:00000200:0.0:1773941372.052212:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941372.052215:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941372.052218:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113dee58 00000400:00000010:0.0:1773941372.052220:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113dee58. 00000100:00000200:0.0:1773941372.052225:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d111f2300 x1860112291466112/t0(0) o8->lustre-OST0001-osc-ffff988d07f1c800@192.168.204.114@tcp:28/4 lens 520/544 e 0 to 0 dl 1773941382 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1773941372.052626:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941372.052632:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941372.052638:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941372.052641:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941372.052645:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941372.052649:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324600c00 00000400:00000200:1.0:1773941372.052652:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941372.052656:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0x341 [1] + 224 00000400:00000200:1.0:1773941372.052667:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941372.052671:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:1.0:1773941372.052677:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10835180 x1860112291466240/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941388 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000010:0.0:1773941372.052735:0:2401:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f7e8. 00000400:00000200:0.0:1773941372.052740:0:2401:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833088 00000400:00000010:0.0:1773941372.052741:0:2401:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833088. 00000100:00000200:0.0:1773941372.052747:0:2401:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10835180 x1860112291466240/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941388 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941372.052779:0:2401:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10835180 x1860112291466240/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941388 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:0.0:1773941372.052795:0:2401:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff988d065c5400. 02000000:00000010:0.0:1773941372.052800:0:2401:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff988d113e9100. 00000100:00000010:0.0:1773941372.052803:0:2401:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10835180. 00000400:00000200:0.0:1773941372.052913:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941372.052918:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941372.052924:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941372.052927:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941372.052930:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941372.052934:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 4 MB=0x69bc324600b00 00000400:00000200:0.0:1773941372.052937:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941372.052942:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.204.114@tcp of length 416/416 into md 0x331 [1] + 0 00000400:00000200:0.0:1773941372.052950:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941372.052954:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:0.0:1773941372.052982:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d111f2a00 x1860112291465984/t0(0) o8->lustre-OST0000-osc-ffff988d07f1c800@192.168.204.114@tcp:28/4 lens 520/544 e 0 to 0 dl 1773941382 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1773941372.053011:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941372.053016:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941372.053022:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941372.053026:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941372.053030:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941372.053033:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 248 into portal 10 MB=0x69bc324600c80 00000400:00000200:0.0:1773941372.053037:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941372.053041:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d877e8. 00000400:00000200:1.0:1773941372.053046:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de990 00000400:00000010:1.0:1773941372.053047:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de990. 00000100:00000200:1.0:1773941372.053054:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d111f2a00 x1860112291465984/t0(0) o8->lustre-OST0000-osc-ffff988d07f1c800@192.168.204.114@tcp:28/4 lens 520/544 e 0 to 0 dl 1773941382 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1773941372.053056:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.114@tcp of length 248/248 into md 0x351 [1] + 224 00000100:00000200:1.0:1773941372.053070:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d111f2a00 x1860112291465984/t0(0) o8->lustre-OST0000-osc-ffff988d07f1c800@192.168.204.114@tcp:28/4 lens 520/544 e 0 to 0 dl 1773941382 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1773941372.053109:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000010:1.0:1773941372.053117:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d1081c800. 00000400:00000200:0.0:1773941372.053118:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 02000000:00000010:1.0:1773941372.053124:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d1127f800. 00000100:00000010:1.0:1773941372.053129:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d111f2a00. 00000100:00000200:0.0:1773941372.053134:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d111f3b80 x1860112291466368/t0(0) o900->lustre-MDT0000-mdc-ffff988d07f1c800@192.168.204.114@tcp:29/10 lens 264/248 e 0 to 0 dl 1773941388 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1773941372.053171:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941372.053176:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941372.053181:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941372.053185:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941372.053191:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941372.053195:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 4 MB=0x69bc324600b80 00000400:00000200:0.0:1773941372.053200:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941372.053206:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.204.114@tcp of length 416/416 into md 0x359 [1] + 0 00000400:00000010:3.0:1773941372.053213:0:6296:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccdb58. 00000400:00000200:3.0:1773941372.053218:0:6296:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac880 00000400:00000200:0.0:1773941372.053219:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000010:3.0:1773941372.053220:0:6296:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac880. 00000400:00000200:0.0:1773941372.053233:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:3.0:1773941372.053235:0:6296:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d111f3b80 x1860112291466368/t0(0) o900->lustre-MDT0000-mdc-ffff988d07f1c800@192.168.204.114@tcp:29/10 lens 264/248 e 0 to 0 dl 1773941388 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941372.053247:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d111f2300 x1860112291466112/t0(0) o8->lustre-OST0001-osc-ffff988d07f1c800@192.168.204.114@tcp:28/4 lens 520/544 e 0 to 0 dl 1773941382 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941372.053248:0:6296:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d111f3b80 x1860112291466368/t0(0) o900->lustre-MDT0000-mdc-ffff988d07f1c800@192.168.204.114@tcp:29/10 lens 264/248 e 0 to 0 dl 1773941388 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1773941372.053291:0:6296:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1056ce40. 02000000:00000010:3.0:1773941372.053297:0:6296:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff988d038bfe00. 02000000:00000010:3.0:1773941372.053303:0:6296:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d038bee00. 00000100:00000010:3.0:1773941372.053314:0:6296:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d111f3b80. 80000000:00000010:3.0:1773941372.053320:0:6296:0:(fld_cache.c:357:fld_cache_entry_create()) kmalloced '(f_new)': 56 at ffff988d07275200. 00000400:00000010:1.0:1773941372.053320:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d87370. 00000400:00000200:1.0:1773941372.053330:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113debb0 00000400:00000010:1.0:1773941372.053332:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113debb0. 00000020:00000010:3.0:1773941372.053334:0:6296:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 640 at ffff988d2036f400. 00000100:00000010:3.0:1773941372.053341:0:6296:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d111f3b80. 00000100:00000200:1.0:1773941372.053341:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d111f2300 x1860112291466112/t0(0) o8->lustre-OST0001-osc-ffff988d07f1c800@192.168.204.114@tcp:28/4 lens 520/544 e 0 to 0 dl 1773941382 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1773941372.053354:0:6296:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d038bee00. 00000100:00000200:1.0:1773941372.053358:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d111f2300 x1860112291466112/t0(0) o8->lustre-OST0001-osc-ffff988d07f1c800@192.168.204.114@tcp:28/4 lens 520/544 e 0 to 0 dl 1773941382 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1773941372.053365:0:6296:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1056ce40. 02000000:00000010:1.0:1773941372.053388:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d1081e400. 02000000:00000010:1.0:1773941372.053393:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d02ce4400. 00000100:00000010:1.0:1773941372.053396:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d111f2300. 02000000:00000010:3.0:1773941372.053436:0:6296:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 131072 at ffff988d18a40000. 00000400:00000200:3.0:1773941372.053441:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941372.053443:0:6296:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccdb58. 00000400:00000010:3.0:1773941372.053447:0:6296:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 632 at ffff988d2036e400. 00000100:00000200:3.0:1773941372.053450:0:6296:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 131072 bytes, xid 1860112291466496, portal 10 00000400:00000010:3.0:1773941372.053454:0:6296:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac880. 00000100:00000200:3.0:1773941372.053456:0:6296:0:(niobuf.c:94:ptl_send_buf()) Sending 448 bytes to portal 23, xid 1860112291466496, offset 0 00000400:00000200:3.0:1773941372.053460:0:6296:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941372.053464:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941372.053467:0:6296:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941372.053470:0:6296:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941372.053473:0:6296:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941372.053475:0:6296:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941372.053477:0:6296:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941372.053479:0:6296:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [30:30:256:4294967295] 192.168.204.114@tcp LPNI seq info [30:30:8:4294967295] 00000400:00000200:3.0:1773941372.053482:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941372.053488:0:6296:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:3.0:1773941372.053491:0:6296:0:(socklnd_cb.c:1001:ksocknal_send()) sending 448 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:3.0:1773941372.053494:0:6296:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d09a9ca00. 00000800:00000200:3.0:1773941372.053498:0:6296:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:3.0:1773941372.053502:0:6296:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:3.0:1773941372.053504:0:6296:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d09a9ca00 type 1, nob 544 niov 1 nkiov 1 00000400:00000200:3.0:1773941372.053517:0:6296:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:1.0:1773941372.053564:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941372.053567:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d09a9ca00. 00000400:00000200:1.0:1773941372.053570:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941372.053574:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941372.053577:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac880 00000400:00000010:1.0:1773941372.053578:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac880. 00000100:00000200:1.0:1773941372.053583:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d111f3b80 x1860112291466496/t0(0) o33->lustre-MDT0000-mdc-ffff988d07f1c800@192.168.204.114@tcp:23/10 lens 448/66264 e 0 to 0 dl 1773941388 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:0 00000400:00000200:0.0:1773941372.071429:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941372.071438:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941372.071447:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941372.071453:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941372.071459:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941372.071463:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 464 into portal 10 MB=0x69bc324600d00 00000400:00000200:0.0:1773941372.071469:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941372.071475:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.114@tcp of length 464/464 into md 0x369 [32] + 224 00000400:00000200:0.0:1773941372.071494:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941372.071500:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:0.0:1773941372.071515:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d111f3b80 x1860112291466496/t0(0) o33->lustre-MDT0000-mdc-ffff988d07f1c800@192.168.204.114@tcp:23/10 lens 448/66264 e 0 to 0 dl 1773941388 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:0 00000400:00000010:3.0:1773941372.071626:0:6296:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccdb58. 00000400:00000200:3.0:1773941372.071641:0:6296:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d2036e400 00000400:00000010:3.0:1773941372.071665:0:6296:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 632 at ffff988d2036e400. 00000100:00000200:3.0:1773941372.071679:0:6296:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d111f3b80 x1860112291466496/t0(0) o33->lustre-MDT0000-mdc-ffff988d07f1c800@192.168.204.114@tcp:23/10 lens 448/66264 e 0 to 0 dl 1773941388 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:0 00000100:00000200:3.0:1773941372.071692:0:6296:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d111f3b80 x1860112291466496/t0(0) o33->lustre-MDT0000-mdc-ffff988d07f1c800@192.168.204.114@tcp:23/10 lens 448/66264 e 0 to 0 dl 1773941388 ref 2 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'' uid:0 gid:0 projid:0 00000100:00000010:3.0:1773941372.071732:0:6296:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1056ce40. 00000002:00000200:3.0:1773941372.071743:0:6296:0:(mdc_request.c:172:mdc_getattr_common()) mode: 40755 00000080:00000010:3.0:1773941372.071750:0:6296:0:(llite_lib.c:753:client_common_fill_super()) kfreed 'op_data': 384 at ffff988d038be000. 00000020:00000010:3.0:1773941372.071781:0:6296:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff988d08a3f3f0. 00000020:00000010:3.0:1773941372.071786:0:6296:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d038be000. 00000020:00000010:3.0:1773941372.071793:0:6296:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d038bfe00. 00000020:00000010:3.0:1773941372.071799:0:6296:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d183509c0. 00000008:00000010:3.0:1773941372.071807:0:6296:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff988d113fd940. 00020000:00000010:3.0:1773941372.071813:0:6296:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff988d20218e40. 00008000:00000010:3.0:1773941372.071820:0:6296:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d107f0710. 00000080:00000010:3.0:1773941372.071828:0:6296:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff988d181e7f18. 00000080:00000010:3.0:1773941372.071834:0:6296:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff988d054ab6c0. 00000020:00000010:3.0:1773941372.071838:0:6296:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d038bf000. 00000008:00000010:3.0:1773941372.071843:0:6296:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff988d035ca000. 00020000:00000010:3.0:1773941372.071848:0:6296:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff988d113f4b40. 00008000:00000010:3.0:1773941372.071853:0:6296:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff988d209199e8. 00000080:00000010:3.0:1773941372.071857:0:6296:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff988d07153000. 00000080:00000010:3.0:1773941372.071863:0:6296:0:(vvp_dev.c:174:vvp_device_alloc()) kmalloced '(vdv)': 72 at ffff988d18350960. 00000080:00000010:3.0:1773941372.071869:0:6296:0:(vvp_dev.c:182:vvp_device_alloc()) kmalloced '(site)': 576 at ffff988d2036e400. 00000020:00000010:3.0:1773941372.071922:0:6296:0:(lu_object.c:1154:lu_site_init()) kmalloced '(s->ls_bkts)': 18432 at ffff988d089b0000. 00000020:00000010:3.0:1773941372.071932:0:6296:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff988d2036d800. 00000020:00000010:3.0:1773941372.071935:0:6296:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 192 at ffff988d1056ce40. 00000008:00000010:3.0:1773941372.071943:0:6296:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff988d113fef90. 00020000:00000010:3.0:1773941372.071945:0:6296:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff988d20218d80. 00008000:00000010:3.0:1773941372.071948:0:6296:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d107f2350. 00000080:00000010:3.0:1773941372.071958:0:6296:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff988d181e72a0. 00000080:00000010:3.0:1773941372.071996:0:6296:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff988d054ab230. 00000008:00000010:3.0:1773941372.072008:0:6296:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff988d035cadc0. 00020000:00000010:3.0:1773941372.072012:0:6296:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff988d113f5d40. 00008000:00000010:3.0:1773941372.072015:0:6296:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff988d20919e90. 00000080:00000010:3.0:1773941372.072018:0:6296:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff988d07153690. 00000008:00000010:3.0:1773941372.072023:0:6296:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff988d113f9650. 00020000:00000010:3.0:1773941372.072026:0:6296:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff988d20218780. 00008000:00000010:3.0:1773941372.072029:0:6296:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d107f11a8. 00000080:00000010:3.0:1773941372.072032:0:6296:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff988d181e70a8. 00000080:00000010:3.0:1773941372.072034:0:6296:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff988d054aa6c8. 00000008:00000010:3.0:1773941372.072038:0:6296:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff988d035cba20. 00020000:00000010:3.0:1773941372.072042:0:6296:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff988d113f6400. 00008000:00000010:3.0:1773941372.072044:0:6296:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff988d20919a50. 00000080:00000010:3.0:1773941372.072047:0:6296:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff988d071531e0. 00000008:00000010:3.0:1773941372.072052:0:6296:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff988d113fc2f0. 00020000:00000010:3.0:1773941372.072055:0:6296:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff988d20218840. 00008000:00000010:3.0:1773941372.072058:0:6296:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d107f3880. 00000080:00000010:3.0:1773941372.072061:0:6296:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff988d181e7540. 00000080:00000010:3.0:1773941372.072064:0:6296:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff988d054ab478. 00000008:00000010:3.0:1773941372.072067:0:6296:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff988d035ca9a0. 00020000:00000010:3.0:1773941372.072070:0:6296:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff988d113f6880. 00008000:00000010:3.0:1773941372.072071:0:6296:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff988d20919268. 00000080:00000010:3.0:1773941372.072073:0:6296:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff988d07153d20. 00000008:00000010:3.0:1773941372.072077:0:6296:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff988d113fa178. 00020000:00000010:3.0:1773941372.072078:0:6296:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff988d20218180. 00008000:00000010:3.0:1773941372.072080:0:6296:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d107f1530. 00000080:00000010:3.0:1773941372.072082:0:6296:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff988d181e75e8. 00000080:00000010:3.0:1773941372.072084:0:6296:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff988d054a8db0. 00000008:00000010:3.0:1773941372.072086:0:6296:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff988d035cb340. 00020000:00000010:3.0:1773941372.072087:0:6296:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff988d113f6640. 00008000:00000010:3.0:1773941372.072088:0:6296:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff988d209193b8. 00000080:00000010:3.0:1773941372.072090:0:6296:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff988d07153780. 00020000:00000010:3.0:1773941372.072098:0:6296:0:(lovsub_dev.c:89:lovsub_device_alloc()) kmalloced '(lsd)': 72 at ffff988d18350a80. 00020000:00000010:3.0:1773941372.072107:0:6296:0:(lovsub_dev.c:89:lovsub_device_alloc()) kmalloced '(lsd)': 72 at ffff988d183507e0. 00020000:00000010:3.0:1773941372.072111:0:6296:0:(lovsub_dev.c:89:lovsub_device_alloc()) kmalloced '(lsd)': 72 at ffff988d18350720. 00020000:00000010:3.0:1773941372.072115:0:6296:0:(lovsub_dev.c:89:lovsub_device_alloc()) kmalloced '(lsd)': 72 at ffff988d183505a0. 00000020:00000010:3.0:1773941372.072120:0:6296:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d038bfe00. 00000020:00000010:3.0:1773941372.072125:0:6296:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d183509c0. 00000008:00000010:3.0:1773941372.072129:0:6296:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff988d113fd940. 00020000:00000010:3.0:1773941372.072133:0:6296:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff988d20218e40. 00008000:00000010:3.0:1773941372.072137:0:6296:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d107f0710. 00000080:00000010:3.0:1773941372.072141:0:6296:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff988d181e7f18. 00000080:00000010:3.0:1773941372.072144:0:6296:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff988d054ab6c0. 00000020:00000010:3.0:1773941372.072148:0:6296:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d038be000. 00000008:00000010:3.0:1773941372.072151:0:6296:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff988d035ca000. 00020000:00000010:3.0:1773941372.072154:0:6296:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff988d113f4b40. 00008000:00000010:3.0:1773941372.072157:0:6296:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff988d209199e8. 00000080:00000010:3.0:1773941372.072160:0:6296:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff988d07153000. 00000020:00000010:3.0:1773941372.072163:0:6296:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d038bf000. 00000020:00000010:3.0:1773941372.072165:0:6296:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff988d08a3f3f0. 00000080:00000010:3.0:1773941372.072200:0:6296:0:(super25.c:43:ll_alloc_inode()) slab-alloced 'lli': 2072 at ffff988d2052a100. 02000000:00000010:3.0:1773941372.072224:0:6296:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 131072 at ffff988d18a40000. 02000000:00000010:3.0:1773941372.072246:0:6296:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d038bee00. 00000100:00000010:3.0:1773941372.072250:0:6296:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d111f3b80. 00000080:00000010:3.0:1773941372.072259:0:6296:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffff988d107cf6e0. 00000080:00000010:3.0:1773941372.072266:0:6296:0:(llite_lib.c:842:client_common_fill_super()) kfreed 'data': 192 at ffff988d05447180. 00000080:00000010:3.0:1773941372.072269:0:6296:0:(llite_lib.c:843:client_common_fill_super()) kfreed 'osfs': 144 at ffff988d05447e40. 00000080:00000010:3.0:1773941372.072300:0:6296:0:(llite_lib.c:1521:ll_fill_super()) kfreed 'md': 33 at ffff988d04d672c0. 00000080:00000010:3.0:1773941372.072304:0:6296:0:(llite_lib.c:1523:ll_fill_super()) kfreed 'dt': 33 at ffff988d04d67400. 00000080:00000010:3.0:1773941372.072308:0:6296:0:(llite_lib.c:1531:ll_fill_super()) kfreed 'cfg': 88 at ffff988d209387e0. 00000080:02000400:3.0:1773941372.072312:0:6296:0:(llite_lib.c:1537:ll_fill_super()) Mounted lustre-client 00000080:00200008:3.0:1773941372.135489:0:6343:0:(file.c:6291:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff988d2052a1c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:3.0:1773941372.135499:0:6343:0:(file.c:6186:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8),name=/ 00000002:00010000:3.0:1773941372.135517:0:6343:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:3.0:1773941372.135574:0:6343:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff988d07f1c800 lock: ffff988d0168b000/0x8e86767622e283a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 6343 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941372.135586:0:6343:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff988d07f1c800 lock: ffff988d0168b000/0x8e86767622e283a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 6343 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941372.135595:0:6343:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff988d07f1c800 lock: ffff988d0168b000/0x8e86767622e283a lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 6343 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1773941372.135629:0:6343:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d111f3b80 pname:cluuid:pid:xid:nid:opc:job mkdir:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:6343:1860112291466624:192.168.204.114@tcp:101: 00000100:00100000:3.0:1773941372.135704:0:6343:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1056c780 going to sleep for 11 seconds 00000100:00100000:3.0:1773941372.137592:0:6343:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d111f3b80 pname:cluuid:pid:xid:nid:opc:job mkdir:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:6343:1860112291466624:192.168.204.114@tcp:101: 00010000:00010000:3.0:1773941372.137610:0:6343:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff988d07f1c800 lock: ffff988d0168b000/0x8e86767622e283a lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd03f0241a6d9429b expref: -99 pid: 6343 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941372.137635:0:6343:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff988d07f1c800 lock: ffff988d0168b000/0x8e86767622e283a lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd03f0241a6d9429b expref: -99 pid: 6343 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941372.137649:0:6343:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff988d07f1c800 lock: ffff988d0168b000/0x8e86767622e283a lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd03f0241a6d9429b expref: -99 pid: 6343 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941372.137663:0:6343:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff988d07f1c800 lock: ffff988d0168b000/0x8e86767622e283a lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd03f0241a6d9429b expref: -99 pid: 6343 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941372.137676:0:6343:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff988d07f1c800 lock: ffff988d0168b000/0x8e86767622e283a lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd03f0241a6d9429b expref: -99 pid: 6343 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941372.137686:0:6343:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff988d07f1c800 lock: ffff988d0168b000/0x8e86767622e283a lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd03f0241a6d9429b expref: -99 pid: 6343 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941372.137707:0:6343:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff988d07f1c800 lock: ffff988d0168b000/0x8e86767622e283a lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd03f0241a6d9429b expref: -99 pid: 6343 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000002:00100000:3.0:1773941372.137722:0:6343:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff988d111f3b80 x1860112291466624/t0(0) o101->lustre-MDT0000-mdc-ffff988d07f1c800@192.168.204.114@tcp:12/10 lens 576/688 e 0 to 0 dl 1773941388 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'' uid:0 gid:0 projid:0 00000002:00010000:3.0:1773941372.137737:0:6343:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff988d07f1c800 lock: ffff988d0168b000/0x8e86767622e283a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd03f0241a6d9429b expref: -99 pid: 6343 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941372.137749:0:6343:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff988d2033d400 type 13 mode 4 res 8589934599/1 (0 0) 00000080:00200000:3.0:1773941372.137755:0:6343:0:(namei.c:951:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00200000:3.0:1773941372.137775:0:6343:0:(llite_lib.c:2979:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 10752 00000080:00010000:3.0:1773941372.137780:0:6343:0:(llite_internal.h:1998:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff988d2052a1c8) for lock 0x8e86767622e283a 00000080:00010000:3.0:1773941372.137786:0:6343:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff988d2052a1c8) 00000080:00010000:3.0:1773941372.137790:0:6343:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x8e86767622e283a from it ffffb6ec83d67cb8 00010000:00010000:3.0:1773941372.137794:0:6343:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff988d07f1c800 lock: ffff988d0168b000/0x8e86767622e283a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd03f0241a6d9429b expref: -99 pid: 6343 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941372.137804:0:6343:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff988d07f1c800 lock: ffff988d0168b000/0x8e86767622e283a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd03f0241a6d9429b expref: -99 pid: 6343 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1773941373.051145:0:6307:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:2.0:1773941373.502161:0:6396:0:(file.c:6291:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff988d2052a1c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:2.0:1773941373.502172:0:6396:0:(file.c:6186:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8),name=/ 00000002:00010000:2.0:1773941373.502196:0:6396:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1773941373.502213:0:6396:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff988d07f1c800 lock: ffff988d0168b000/0x8e86767622e283a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd03f0241a6d9429b expref: -99 pid: 6343 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1773941373.502227:0:6396:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff988d07f1c800 lock: ffff988d0168b000/0x8e86767622e283a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd03f0241a6d9429b expref: -99 pid: 6343 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000080:00200000:2.0:1773941373.502241:0:6396:0:(namei.c:951:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1773941373.502247:0:6396:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff988d2052a1c8) 00000080:00010000:2.0:1773941373.502250:0:6396:0:(llite_internal.h:1998:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff988d2052a1c8) for lock 0x8e86767622e283a 00000080:00010000:2.0:1773941373.502257:0:6396:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x8e86767622e283a from it ffffb6ec818dbcb8 00010000:00010000:2.0:1773941373.502260:0:6396:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff988d07f1c800 lock: ffff988d0168b000/0x8e86767622e283a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd03f0241a6d9429b expref: -99 pid: 6343 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1773941373.502267:0:6396:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff988d07f1c800 lock: ffff988d0168b000/0x8e86767622e283a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd03f0241a6d9429b expref: -99 pid: 6343 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00000080:00200008:2.0:1773941373.502753:0:6396:0:(file.c:6291:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff988d2052a1c8), request_mask 0, flags 0, foreign 0 00000080:00200000:2.0:1773941373.502757:0:6396:0:(file.c:6186:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8),name=/ 00000002:00010000:2.0:1773941373.502764:0:6396:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1773941373.502771:0:6396:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff988d07f1c800 lock: ffff988d0168b000/0x8e86767622e283a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd03f0241a6d9429b expref: -99 pid: 6343 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00010000:00010000:2.0:1773941373.502778:0:6396:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff988d07f1c800 lock: ffff988d0168b000/0x8e86767622e283a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd03f0241a6d9429b expref: -99 pid: 6343 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00000080:00200000:2.0:1773941373.502783:0:6396:0:(namei.c:951:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1773941373.502786:0:6396:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff988d2052a1c8) 00000080:00010000:2.0:1773941373.502788:0:6396:0:(llite_internal.h:1998:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff988d2052a1c8) for lock 0x8e86767622e283a 00000080:00010000:2.0:1773941373.502791:0:6396:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x8e86767622e283a from it ffffb6ec818dbcb8 00010000:00010000:2.0:1773941373.502793:0:6396:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff988d07f1c800 lock: ffff988d0168b000/0x8e86767622e283a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd03f0241a6d9429b expref: -99 pid: 6343 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00010000:00010000:2.0:1773941373.502806:0:6396:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff988d07f1c800 lock: ffff988d0168b000/0x8e86767622e283a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd03f0241a6d9429b expref: -99 pid: 6343 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00000080:00200000:2.0:1773941373.502885:0:6396:0:(file.c:6186:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8),name=/ 00000002:00010000:2.0:1773941373.502891:0:6396:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:2.0:1773941373.502897:0:6396:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff988d07f1c800 lock: ffff988d0168b000/0x8e86767622e283a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd03f0241a6d9429b expref: -99 pid: 6343 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00010000:00010000:2.0:1773941373.502903:0:6396:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff988d07f1c800 lock: ffff988d0168b000/0x8e86767622e283a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd03f0241a6d9429b expref: -99 pid: 6343 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00000080:00200000:2.0:1773941373.502907:0:6396:0:(namei.c:951:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:2.0:1773941373.502910:0:6396:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff988d2052a1c8) 00000080:00010000:2.0:1773941373.502912:0:6396:0:(llite_internal.h:1998:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff988d2052a1c8) for lock 0x8e86767622e283a 00000080:00010000:2.0:1773941373.502915:0:6396:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x8e86767622e283a from it ffffb6ec818dbbb0 00010000:00010000:2.0:1773941373.502917:0:6396:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff988d07f1c800 lock: ffff988d0168b000/0x8e86767622e283a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd03f0241a6d9429b expref: -99 pid: 6343 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00010000:00010000:2.0:1773941373.502922:0:6396:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff988d07f1c800 lock: ffff988d0168b000/0x8e86767622e283a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd03f0241a6d9429b expref: -99 pid: 6343 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00000080:00200000:2.0:1773941373.502927:0:6396:0:(file.c:6668:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8), inode mode 41ed mask 44 00000080:00200008:2.0:1773941373.502932:0:6396:0:(file.c:1025:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff988d2052a1c8), flags 100000 00000080:00200080:2.0:1773941373.503036:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.503052:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00800000:00000004:2.0:1773941373.503060:0:6396:0:(obd_class.h:1083:obd_statfs()) lustre-MDT0000-mdc-ffff988d07f1c800: age 95, max_age 95 00800000:00000004:2.0:1773941373.503069:0:6396:0:(obd_class.h:1123:obd_statfs()) lustre-MDT0000-mdc-ffff988d07f1c800: use ffff988d089a8450 cache blocks 17230/17271 objects 502289/502621 00000080:00200080:2.0:1773941373.503098:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.503106:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00800000:00000004:2.0:1773941373.503111:0:6396:0:(obd_class.h:1083:obd_statfs()) lustre-MDT0001-mdc-ffff988d07f1c800: age -906, max_age 95 00000100:00100000:2.0:1773941373.503154:0:6396:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06504000 pname:cluuid:pid:xid:nid:opc:job lfs:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:6396:1860112291466752:192.168.204.114@tcp:41: 00000100:00100000:2.0:1773941373.503267:0:6396:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d08feb780 going to sleep for 11 seconds 00010000:00080000:2.0:1773941373.504481:0:6396:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff988d06504000 x1860112291466752/t0(0) o41->lustre-MDT0001-mdc-ffff988d07f1c800@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941389 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941373.504495:0:6396:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06504000 pname:cluuid:pid:xid:nid:opc:job lfs:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:6396:1860112291466752:192.168.204.114@tcp:41: 00800000:00000004:2.0:1773941373.504510:0:6396:0:(obd_class.h:1109:obd_statfs()) lustre-MDT0001-mdc-ffff988d07f1c800: update ffff988d2039b7c0 cache blocks 17232/17271 objects 503695/504009 00000080:00200080:2.0:1773941373.504541:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504549:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504554:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504557:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504561:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504563:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504567:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504570:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504573:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504575:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504578:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504580:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504582:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504585:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504603:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504614:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504618:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504622:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504625:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504627:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504630:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504632:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504636:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504638:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504642:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504644:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504647:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504650:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504653:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504656:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504658:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504661:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504663:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504667:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504669:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504672:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504674:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504677:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504680:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504682:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504690:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504694:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504697:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504700:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504702:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504705:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504708:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504711:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504714:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504716:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504718:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504721:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504724:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504726:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504730:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504732:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504735:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504737:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504740:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504742:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504745:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504747:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504749:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504751:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504754:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504761:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504765:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504767:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504769:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504771:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504773:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504775:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504779:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504781:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504784:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504787:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504789:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504792:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504794:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504796:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504798:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504800:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504803:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504805:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504807:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504809:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504811:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504813:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504815:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504817:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504824:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504827:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504829:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504831:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504833:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504835:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504837:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504839:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504842:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504844:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504846:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504848:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504850:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504852:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504854:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504856:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504858:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504860:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504862:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504864:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504867:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504869:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504871:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504873:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504875:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504881:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504884:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504886:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504888:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504890:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504892:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504894:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504896:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504898:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504901:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00800000:00000080:2.0:1773941373.504903:0:6396:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.504905:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.504909:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00020000:00000004:2.0:1773941373.504915:0:6396:0:(obd_class.h:1083:obd_statfs()) lustre-OST0000-osc-ffff988d07f1c800: age -906, max_age 95 00000100:00100000:2.0:1773941373.504931:0:6396:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06504000 pname:cluuid:pid:xid:nid:opc:job lfs:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:6396:1860112291466880:192.168.204.114@tcp:13: 00000100:00100000:2.0:1773941373.504996:0:6396:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d08feb780 going to sleep for 11 seconds 00010000:00080000:2.0:1773941373.505723:0:6396:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff988d06504000 x1860112291466880/t0(0) o13->lustre-OST0000-osc-ffff988d07f1c800@192.168.204.114@tcp:7/4 lens 224/368 e 0 to 0 dl 1773941389 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941373.505732:0:6396:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06504000 pname:cluuid:pid:xid:nid:opc:job lfs:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:6396:1860112291466880:192.168.204.114@tcp:13: 00020000:00000004:2.0:1773941373.505740:0:6396:0:(obd_class.h:1109:obd_statfs()) lustre-OST0000-osc-ffff988d07f1c800: update ffff988d20398450 cache blocks 3678/3683 objects 117760/118218 00000080:00200080:2.0:1773941373.505768:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.505774:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00020000:00000004:2.0:1773941373.505777:0:6396:0:(obd_class.h:1083:obd_statfs()) lustre-OST0001-osc-ffff988d07f1c800: age -906, max_age 95 00000100:00100000:2.0:1773941373.505789:0:6396:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06504000 pname:cluuid:pid:xid:nid:opc:job lfs:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:6396:1860112291467008:192.168.204.114@tcp:13: 00000100:00100000:2.0:1773941373.505840:0:6396:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d08feb780 going to sleep for 11 seconds 00010000:00080000:2.0:1773941373.506526:0:6396:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff988d06504000 x1860112291467008/t0(0) o13->lustre-OST0001-osc-ffff988d07f1c800@192.168.204.114@tcp:7/4 lens 224/368 e 0 to 0 dl 1773941389 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941373.506535:0:6396:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06504000 pname:cluuid:pid:xid:nid:opc:job lfs:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:6396:1860112291467008:192.168.204.114@tcp:13: 00020000:00000004:2.0:1773941373.506543:0:6396:0:(obd_class.h:1109:obd_statfs()) lustre-OST0001-osc-ffff988d07f1c800: update ffff988d20399e08 cache blocks 3678/3683 objects 117760/118086 00000080:00200080:2.0:1773941373.506569:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506575:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506579:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506582:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506599:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506604:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506609:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506612:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506616:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506619:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506622:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506624:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506626:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506629:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506633:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506635:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506638:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506641:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506643:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506653:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506657:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506660:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506663:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506666:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506669:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506671:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506674:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506677:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506680:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506682:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506685:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506688:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506691:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506693:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506696:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506699:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506703:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506705:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506707:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506710:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506712:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506715:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506718:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506720:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506728:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506731:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506734:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506736:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506739:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506742:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506745:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506748:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506751:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506754:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506757:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506759:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506761:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506763:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506766:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506767:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506770:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506772:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506775:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506779:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506782:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506784:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506787:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506791:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506795:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506801:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506805:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506808:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506811:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506813:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506816:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506819:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506821:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506823:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506825:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506827:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506829:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506831:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506834:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506835:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506838:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506840:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506842:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506844:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506846:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506848:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506850:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506852:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506854:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506856:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506862:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506865:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506867:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506869:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506871:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506873:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506876:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506877:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506880:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506882:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506884:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506886:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506888:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506890:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506892:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506894:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506896:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506898:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506900:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506902:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506904:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506906:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506908:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506910:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506912:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506918:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506921:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506923:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506925:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506927:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200080:2.0:1773941373.506929:0:6396:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) cmd=c00866a4 arg=7ffed24ca1a0 00020000:00000080:2.0:1773941373.506931:0:6396:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=c00866a4 len=592 karg=ffff988d203e4400 uarg=0000000000000000 00000080:00200008:2.0:1773941373.506941:0:6396:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff988d2052a1c8), flags 100000 00010000:00010000:3.0:1773941374.075069:0:6308:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:3.0:1773941374.408123:0:6589:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: Using TIMEOUT=20 00010000:00010000:3.0:1773941375.099094:0:6307:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1773941375.658852:0:2394:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.114@tcp (source: 12345-192.168.204.114@tcp) 00000100:00100000:0.0:1773941375.658902:0:2394:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.114@tcp (source: 12345-192.168.204.114@tcp) 00000100:00100000:1.0:1773941375.658942:0:6297:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1860112298844800 00000100:00100000:0.0:1773941375.658942:0:6298:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1860112298845184 00000100:00100000:1.0:1773941375.658955:0:6297:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1860112298844800 00000100:00100000:0.0:1773941375.658955:0:6298:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1860112298845184 00000100:00100000:1.0:1773941375.659057:0:6297:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.114@tcp, seq: 0 00000100:00100000:1.0:1773941375.659069:0:6297:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff988d10836a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:69e55c94-95bd-430e-b45e-3e120ab74187+4:9552:x1860112298845184:12345-192.168.204.114@tcp:104: 00010000:00010000:1.0:1773941375.659088:0:6297:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.204.114@tcp lock: ffff988d01489800/0x8e86767622e27ca lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400000000000 nid: local remote: 0xd03f0241a6d941a6 expref: -99 pid: 6296 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941375.659172:0:6297:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1773941375.659185:0:6297:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff988d10836a00 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:69e55c94-95bd-430e-b45e-3e120ab74187+4:9552:x1860112298845184:12345-192.168.204.114@tcp:104: Request processed in 118us (282us total) trans 0 rc 0/0 00000100:00100000:1.0:1773941375.659196:0:6297:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.114@tcp, seq: 0 00010000:00010000:3.0:1773941375.659212:0:6308:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00000100:00100000:1.0:1773941375.659220:0:6297:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.114@tcp, seq: 1 00010000:00010000:3.0:1773941375.659223:0:6308:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d01489800/0x8e86767622e27ca lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400000000000 nid: local remote: 0xd03f0241a6d941a6 expref: -99 pid: 6296 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1773941375.659232:0:6297:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff988d10835180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:69e55c94-95bd-430e-b45e-3e120ab74187+3:9552:x1860112298844800:12345-192.168.204.114@tcp:104: 00010000:00010000:1.0:1773941375.659243:0:6297:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.204.114@tcp lock: ffff988d01488000/0x8e86767622e27d1 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x1400000000000 nid: local remote: 0xd03f0241a6d941ad expref: -99 pid: 6296 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941375.659264:0:6308:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d01489800 already unused, calling callback (ffffffffc12b4c20) 10000000:00010000:3.0:1773941375.659271:0:6308:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d01489800/0x8e86767622e27ca lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1400400000000 nid: local remote: 0xd03f0241a6d941a6 expref: -99 pid: 6296 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941375.659283:0:6308:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d01489800/0x8e86767622e27ca lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1408400000000 nid: local remote: 0xd03f0241a6d941a6 expref: -99 pid: 6296 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:3.0:1773941375.659292:0:6308:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d01489800/0x8e86767622e27ca lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1409400000000 nid: local remote: 0xd03f0241a6d941a6 expref: -99 pid: 6296 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941375.659309:0:6297:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1773941375.659316:0:6297:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff988d10835180 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_000:69e55c94-95bd-430e-b45e-3e120ab74187+3:9552:x1860112298844800:12345-192.168.204.114@tcp:104: Request processed in 89us (477us total) trans 0 rc 0/0 00000100:00100000:1.0:1773941375.659323:0:6297:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.114@tcp, seq: 1 00010000:00010000:2.0:1773941375.659360:0:6307:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 10000000:01000000:3.0:1773941375.659380:0:6308:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00010000:3.0:1773941375.659499:0:6308:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: MGC192.168.204.114@tcp lock: ffff988d01489800/0x8e86767622e27ca lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5c09400000020 nid: local remote: 0xd03f0241a6d941a6 expref: -99 pid: 6296 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941375.659508:0:6308:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1773941375.659537:0:6308:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d01489800/0x8e86767622e27ca lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5c09400000020 nid: local remote: 0xd03f0241a6d941a6 expref: -99 pid: 6296 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941375.659561:0:6308:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d01489800/0x8e86767622e27ca lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5c09400000020 nid: local remote: 0xd03f0241a6d941a6 expref: -99 pid: 6296 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941375.659581:0:6308:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1773941375.659584:0:6308:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1773941375.659588:0:6308:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1773941375.659638:0:6781:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1773941375.659641:0:6781:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1773941375.659643:0:6781:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1773941375.659666:0:6307:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d01488000/0x8e86767622e27d1 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0xd03f0241a6d941ad expref: -99 pid: 6296 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941375.659675:0:6307:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d01488000 already unused, calling callback (ffffffffc12b4c20) 10000000:00010000:0.0:1773941375.659678:0:6307:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d01488000/0x8e86767622e27d1 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1400400000000 nid: local remote: 0xd03f0241a6d941ad expref: -99 pid: 6296 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941375.659684:0:6307:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d01488000/0x8e86767622e27d1 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1408400000000 nid: local remote: 0xd03f0241a6d941ad expref: -99 pid: 6296 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:0.0:1773941375.659689:0:6307:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d01488000/0x8e86767622e27d1 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1409400000000 nid: local remote: 0xd03f0241a6d941ad expref: -99 pid: 6296 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1773941375.659696:0:6307:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000100:00100000:2.0:1773941375.659703:0:2404:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d111f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:69e55c94-95bd-430e-b45e-3e120ab74187:2404:1860112291467136:192.168.204.114@tcp:103: 00010000:00010000:0.0:1773941375.659725:0:6307:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: MGC192.168.204.114@tcp lock: ffff988d01488000/0x8e86767622e27d1 lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0xd03f0241a6d941ad expref: -99 pid: 6296 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941375.659735:0:6307:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:0.0:1773941375.659755:0:6307:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d01488000/0x8e86767622e27d1 lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0xd03f0241a6d941ad expref: -99 pid: 6296 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941375.659765:0:6307:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d01488000/0x8e86767622e27d1 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0xd03f0241a6d941ad expref: -99 pid: 6296 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1773941375.659769:0:2401:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10836300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:69e55c94-95bd-430e-b45e-3e120ab74187:2401:1860112291467264:192.168.204.114@tcp:103: 00010000:00010000:0.0:1773941375.659795:0:6307:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1773941375.659799:0:6307:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1773941375.659802:0:6307:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1773941375.660588:0:2404:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d111f3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:69e55c94-95bd-430e-b45e-3e120ab74187:2404:1860112291467136:192.168.204.114@tcp:103: 00000100:00100000:1.0:1773941375.660748:0:2401:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10836300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:69e55c94-95bd-430e-b45e-3e120ab74187:2401:1860112291467264:192.168.204.114@tcp:103: 00010000:00010000:3.0:1773941376.123105:0:6308:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:01000000:1.0:1773941376.315087:0:6310:0:(mgc_request.c:611:do_requeue()) updating log lustre-client 10000000:01000000:1.0:1773941376.315097:0:6310:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-client-ffff988d07f1c800 from 48 10000000:01000000:1.0:1773941376.315103:0:6310:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-client (res 0x65727473756c) 00010000:00010000:1.0:1773941376.315151:0:6310:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2841 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941376.315164:0:6310:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2841 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941376.315177:0:6310:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2841 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1773941376.315227:0:6310:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:69e55c94-95bd-430e-b45e-3e120ab74187:6310:1860112291467392:192.168.204.114@tcp:101: 00000100:00100000:1.0:1773941376.315318:0:6310:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d08feb480 going to sleep for 11 seconds 00000100:00100000:1.0:1773941376.316687:0:6310:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:69e55c94-95bd-430e-b45e-3e120ab74187:6310:1860112291467392:192.168.204.114@tcp:101: 00010000:00010000:1.0:1773941376.316706:0:6310:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2841 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xd03f0241a6d942a9 expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941376.316725:0:6310:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2841 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0xd03f0241a6d942a9 expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1773941376.316782:0:6310:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:69e55c94-95bd-430e-b45e-3e120ab74187:6310:1860112291467520:192.168.204.114@tcp:501: 00000100:00100000:1.0:1773941376.316823:0:6310:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d08feb480 going to sleep for 11 seconds 00000100:00100000:1.0:1773941376.317654:0:6310:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:69e55c94-95bd-430e-b45e-3e120ab74187:6310:1860112291467520:192.168.204.114@tcp:501: 00000100:00100000:1.0:1773941376.317675:0:6310:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:69e55c94-95bd-430e-b45e-3e120ab74187:6310:1860112291467648:192.168.204.114@tcp:503: 00000100:00100000:1.0:1773941376.317718:0:6310:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d08feb480 going to sleep for 11 seconds 00000100:00100000:1.0:1773941376.318649:0:6310:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:69e55c94-95bd-430e-b45e-3e120ab74187:6310:1860112291467648:192.168.204.114@tcp:503: 00000100:00100000:2.0:1773941376.319020:0:6790:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06504000 pname:cluuid:pid:xid:nid:opc:job llog_process_th:69e55c94-95bd-430e-b45e-3e120ab74187:6790:1860112291467776:192.168.204.114@tcp:502: 00000100:00100000:2.0:1773941376.319083:0:6790:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d10879e40 going to sleep for 11 seconds 00000100:00100000:2.0:1773941376.320065:0:6790:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06504000 pname:cluuid:pid:xid:nid:opc:job llog_process_th:69e55c94-95bd-430e-b45e-3e120ab74187:6790:1860112291467776:192.168.204.114@tcp:502: 00000020:01000000:2.0:1773941376.320092:0:6790:0:(obd_config.c:1923:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:2.0:1773941376.320102:0:6790:0:(obd_config.c:1506:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1773941376.320104:0:6790:0:(obd_config.c:1594:class_process_config()) marker 40 (0x1) lustre sys.jobid_var 00000020:00000080:2.0:1773941376.320110:0:6790:0:(obd_config.c:1506:class_process_config()) processing cmd: cf00f 00000100:00100000:2.0:1773941376.320131:0:6790:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06504000 pname:cluuid:pid:xid:nid:opc:job llog_process_th:69e55c94-95bd-430e-b45e-3e120ab74187:6790:1860112291467904:192.168.204.114@tcp:502:llog_process_th.0 00000100:00100000:2.0:1773941376.320170:0:6790:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d10879e40 going to sleep for 11 seconds 00000100:00100000:2.0:1773941376.321066:0:6790:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06504000 pname:cluuid:pid:xid:nid:opc:job llog_process_th:69e55c94-95bd-430e-b45e-3e120ab74187:6790:1860112291467904:192.168.204.114@tcp:502:llog_process_th.0 00000020:01000000:2.0:1773941376.321090:0:6790:0:(obd_config.c:1923:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:2.0:1773941376.321096:0:6790:0:(obd_config.c:1506:class_process_config()) processing cmd: cf010 00000020:00000080:2.0:1773941376.321098:0:6790:0:(obd_config.c:1594:class_process_config()) marker 40 (0x2) lustre sys.jobid_var 00000040:00080000:2.0:1773941376.321108:0:6790:0:(llog.c:854:llog_process_thread()) stop processing plain [0xa:0x3:0x0] index 52 count 51 00000020:01000000:1.0:1773941376.321209:0:6310:0:(obd_config.c:2178:class_config_parse_llog()) Processed log lustre-client gen 48-51 (rc=0) 10000000:01000000:1.0:1773941376.321228:0:6310:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-client' succeeded (0). 00010000:00010000:1.0:1773941376.321237:0:6310:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2841 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xd03f0241a6d942a9 expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941376.321250:0:6310:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2841 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xd03f0241a6d942a9 expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1773941376.321259:0:6310:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:01000000:1.0:1773941376.321269:0:6310:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d089a99b8 from 1 10000000:01000000:1.0:1773941376.321272:0:6310:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00010000:00010000:1.0:1773941376.321295:0:6310:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488000/0x8e86767622e2848 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941376.321305:0:6310:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d01488000/0x8e86767622e2848 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941376.321315:0:6310:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d01488000/0x8e86767622e2848 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1773941376.321331:0:6310:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:69e55c94-95bd-430e-b45e-3e120ab74187:6310:1860112291468032:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00100000:1.0:1773941376.321373:0:6310:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d08feb480 going to sleep for 11 seconds 00000100:00100000:1.0:1773941376.322243:0:6310:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:69e55c94-95bd-430e-b45e-3e120ab74187:6310:1860112291468032:192.168.204.114@tcp:101:ll_cfg_requeue.0 00010000:00010000:1.0:1773941376.322259:0:6310:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.114@tcp lock: ffff988d01488000/0x8e86767622e2848 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0xd03f0241a6d942b0 expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941376.322269:0:6310:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.114@tcp lock: ffff988d01488000/0x8e86767622e2848 lrc: 4/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0xd03f0241a6d942b0 expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1773941376.322313:0:6310:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:69e55c94-95bd-430e-b45e-3e120ab74187:6310:1860112291468160:192.168.204.114@tcp:501:ll_cfg_requeue.0 00000100:00100000:1.0:1773941376.322355:0:6310:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d08feb480 going to sleep for 11 seconds 00000100:00100000:1.0:1773941376.323116:0:6310:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:69e55c94-95bd-430e-b45e-3e120ab74187:6310:1860112291468160:192.168.204.114@tcp:501:ll_cfg_requeue.0 10000000:01000000:1.0:1773941376.323141:0:6310:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-2). 00010000:00010000:1.0:1773941376.323147:0:6310:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488000/0x8e86767622e2848 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0xd03f0241a6d942b0 expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941376.323155:0:6310:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.114@tcp lock: ffff988d01488000/0x8e86767622e2848 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0xd03f0241a6d942b0 expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1773941377.011713:0:2394:0:(events.c:340:request_in_callback()) peer: 12345-192.168.204.114@tcp (source: 12345-192.168.204.114@tcp) 00000100:00100000:1.0:1773941377.011782:0:6298:0:(service.c:2280:ptlrpc_server_handle_req_in()) unwrap req x1860112298846208 00000100:00100000:1.0:1773941377.011792:0:6298:0:(service.c:2345:ptlrpc_server_handle_req_in()) got req x1860112298846208 00000100:00100000:1.0:1773941377.011842:0:6298:0:(nrs_fifo.c:152:nrs_fifo_req_get()) NRS start fifo request from 12345-192.168.204.114@tcp, seq: 2 00000100:00100000:1.0:1773941377.011852:0:6298:0:(service.c:2531:ptlrpc_server_handle_request()) Handling RPC req@ffff988d10837b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:69e55c94-95bd-430e-b45e-3e120ab74187+3:9605:x1860112298846208:12345-192.168.204.114@tcp:104: 00010000:00010000:1.0:1773941377.011869:0:6298:0:(ldlm_lockd.c:2506:ldlm_callback_handler()) ### blocking ast ns: MGC192.168.204.114@tcp lock: ffff988d0168b600/0x8e86767622e2809 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0xd03f0241a6d941bb expref: -99 pid: 6296 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941377.011937:0:6298:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to priority blp list, 1 blwis in pool 00000100:00100000:1.0:1773941377.011943:0:6298:0:(service.c:2589:ptlrpc_server_handle_request()) Handled RPC req@ffff988d10837b80 pname:cluuid+ref:pid:xid:nid:opc:job ldlm_cb00_001:69e55c94-95bd-430e-b45e-3e120ab74187+3:9605:x1860112298846208:12345-192.168.204.114@tcp:104: Request processed in 93us (238us total) trans 0 rc 0/0 00000100:00100000:1.0:1773941377.011949:0:6298:0:(nrs_fifo.c:211:nrs_fifo_req_stop()) NRS stop fifo request from 12345-192.168.204.114@tcp, seq: 2 00010000:00010000:2.0:1773941377.011985:0:6781:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1773941377.011994:0:6781:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d0168b600/0x8e86767622e2809 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1400000000000 nid: local remote: 0xd03f0241a6d941bb expref: -99 pid: 6296 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941377.012002:0:6781:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d0168b600 already unused, calling callback (ffffffffc12b4c20) 10000000:00010000:2.0:1773941377.012006:0:6781:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d0168b600/0x8e86767622e2809 lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1400400000000 nid: local remote: 0xd03f0241a6d941bb expref: -99 pid: 6296 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941377.012014:0:6781:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d0168b600/0x8e86767622e2809 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1408400000000 nid: local remote: 0xd03f0241a6d941bb expref: -99 pid: 6296 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:2.0:1773941377.012020:0:6781:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d0168b600/0x8e86767622e2809 lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1409400000000 nid: local remote: 0xd03f0241a6d941bb expref: -99 pid: 6296 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1773941377.012024:0:6781:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x736d61726170:0x3:0x0].0x0 (params) 00010000:00010000:2.0:1773941377.012063:0:6781:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: MGC192.168.204.114@tcp lock: ffff988d0168b600/0x8e86767622e2809 lrc: 2/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0xd03f0241a6d941bb expref: -99 pid: 6296 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941377.012068:0:6781:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:2.0:1773941377.012090:0:6781:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d0168b600/0x8e86767622e2809 lrc: 1/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0xd03f0241a6d941bb expref: -99 pid: 6296 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941377.012096:0:6781:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d0168b600/0x8e86767622e2809 lrc: 0/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5c09400000020 nid: local remote: 0xd03f0241a6d941bb expref: -99 pid: 6296 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941377.012111:0:6781:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1773941377.012113:0:6781:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1773941377.012114:0:6781:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1773941377.012121:0:2404:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1773941377.012133:0:2404:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06504000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:69e55c94-95bd-430e-b45e-3e120ab74187:2404:1860112291468288:192.168.204.114@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1773941377.013158:0:2404:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06504000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:69e55c94-95bd-430e-b45e-3e120ab74187:2404:1860112291468288:192.168.204.114@tcp:103:ldlm_bl.0 00010000:00010000:0.0:1773941377.147079:0:6307:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1773941377.275240:0:2403:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d111f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:2403:1860112291468544:192.168.204.114@tcp:400:kworker.0 00000100:00100000:2.0:1773941377.275242:0:2404:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d111f1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:2404:1860112291468416:192.168.204.114@tcp:400:kworker.0 00000100:00100000:2.0:1773941377.275360:0:2404:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d111f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:2404:1860112291468672:192.168.204.114@tcp:400:kworker.0 00000100:00100000:3.0:1773941377.275367:0:2403:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d111f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:2403:1860112291468800:192.168.204.114@tcp:400:kworker.0 00000100:00100000:2.0:1773941377.276676:0:2404:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d111f1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:2404:1860112291468416:192.168.204.114@tcp:400:kworker.0 00010000:00080000:3.0:1773941377.276698:0:2403:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff988d111f2680 x1860112291468544/t0(0) o400->lustre-MDT0001-mdc-ffff988d07f1c800@192.168.204.114@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941393 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00080000:2.0:1773941377.276699:0:2404:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff988d111f1f80 x1860112291468672/t0(0) o400->lustre-OST0000-osc-ffff988d07f1c800@192.168.204.114@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941393 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941377.276743:0:2404:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d111f1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:2404:1860112291468672:192.168.204.114@tcp:400:kworker.0 00000100:00100000:3.0:1773941377.276756:0:2403:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d111f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:2403:1860112291468544:192.168.204.114@tcp:400:kworker.0 00010000:00080000:3.0:1773941377.276785:0:2403:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff988d111f2d80 x1860112291468800/t0(0) o400->lustre-OST0001-osc-ffff988d07f1c800@192.168.204.114@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941393 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941377.276794:0:2403:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d111f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:2403:1860112291468800:192.168.204.114@tcp:400:kworker.0 10000000:01000000:1.0:1773941377.915069:0:6310:0:(mgc_request.c:611:do_requeue()) updating log params 10000000:01000000:1.0:1773941377.915079:0:6310:0:(mgc_request.c:1813:mgc_process_log()) Process log params-ffff988d07f1c800 from 4 10000000:01000000:1.0:1773941377.915086:0:6310:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for params (res 0x736d61726170) 00010000:00010000:1.0:1773941377.915144:0:6310:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e284f lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941377.915157:0:6310:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e284f lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941377.915170:0:6310:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e284f lrc: 3/1,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1773941377.915198:0:6310:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:69e55c94-95bd-430e-b45e-3e120ab74187:6310:1860112291468928:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00100000:1.0:1773941377.915298:0:6310:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d055f5180 going to sleep for 11 seconds 00000100:00100000:1.0:1773941377.916660:0:6310:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:69e55c94-95bd-430e-b45e-3e120ab74187:6310:1860112291468928:192.168.204.114@tcp:101:ll_cfg_requeue.0 00010000:00010000:1.0:1773941377.916681:0:6310:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e284f lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xd03f0241a6d942be expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941377.916707:0:6310:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e284f lrc: 4/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0xd03f0241a6d942be expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:1.0:1773941377.916759:0:6310:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:69e55c94-95bd-430e-b45e-3e120ab74187:6310:1860112291469056:192.168.204.114@tcp:501:ll_cfg_requeue.0 00000100:00100000:1.0:1773941377.916789:0:6310:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d055f5180 going to sleep for 11 seconds 00000100:00100000:1.0:1773941377.917567:0:6310:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:69e55c94-95bd-430e-b45e-3e120ab74187:6310:1860112291469056:192.168.204.114@tcp:501:ll_cfg_requeue.0 00000100:00100000:1.0:1773941377.917601:0:6310:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:69e55c94-95bd-430e-b45e-3e120ab74187:6310:1860112291469184:192.168.204.114@tcp:503:ll_cfg_requeue.0 00000100:00100000:1.0:1773941377.917648:0:6310:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d055f5180 going to sleep for 11 seconds 00000100:00100000:1.0:1773941377.918440:0:6310:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0380 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:69e55c94-95bd-430e-b45e-3e120ab74187:6310:1860112291469184:192.168.204.114@tcp:503:ll_cfg_requeue.0 00000100:00100000:3.0:1773941377.918733:0:6854:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d111f2d80 pname:cluuid:pid:xid:nid:opc:job llog_process_th:69e55c94-95bd-430e-b45e-3e120ab74187:6854:1860112291469312:192.168.204.114@tcp:502:llog_process_th.0 00000100:00100000:3.0:1773941377.918774:0:6854:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a9dd900 going to sleep for 11 seconds 00000100:00100000:3.0:1773941377.919549:0:6854:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d111f2d80 pname:cluuid:pid:xid:nid:opc:job llog_process_th:69e55c94-95bd-430e-b45e-3e120ab74187:6854:1860112291469312:192.168.204.114@tcp:502:llog_process_th.0 00000020:01000000:3.0:1773941377.919576:0:6854:0:(obd_config.c:1923:class_config_llog_handler()) Marker, inst_flg=0x0 mark_flg=0x1 00000020:00000080:3.0:1773941377.919585:0:6854:0:(obd_config.c:1506:class_process_config()) processing cmd: cf010 00000020:00000080:3.0:1773941377.919588:0:6854:0:(obd_config.c:1594:class_process_config()) marker 3 (0x1) general lod.*.mdt_hash 00000020:01000000:3.0:1773941377.919594:0:6854:0:(obd_config.c:2038:class_config_llog_handler()) cmd ce032, instance name: general-ffff988d07f1c800 00000020:00000080:3.0:1773941377.919598:0:6854:0:(obd_config.c:1506:class_process_config()) processing cmd: ce032 00000100:00100000:3.0:1773941377.919659:0:6854:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d111f2d80 pname:cluuid:pid:xid:nid:opc:job llog_process_th:69e55c94-95bd-430e-b45e-3e120ab74187:6854:1860112291469440:192.168.204.114@tcp:502:llog_process_th.0 00000100:00100000:3.0:1773941377.919697:0:6854:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a9dd900 going to sleep for 11 seconds 00000100:00100000:3.0:1773941377.920619:0:6854:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d111f2d80 pname:cluuid:pid:xid:nid:opc:job llog_process_th:69e55c94-95bd-430e-b45e-3e120ab74187:6854:1860112291469440:192.168.204.114@tcp:502:llog_process_th.0 00000020:01000000:3.0:1773941377.920641:0:6854:0:(obd_config.c:1923:class_config_llog_handler()) Marker, inst_flg=0x2 mark_flg=0x2 00000020:00000080:3.0:1773941377.920646:0:6854:0:(obd_config.c:1506:class_process_config()) processing cmd: cf010 00000020:00000080:3.0:1773941377.920648:0:6854:0:(obd_config.c:1594:class_process_config()) marker 3 (0x2) general lod.*.mdt_hash 00000040:00080000:3.0:1773941377.920651:0:6854:0:(llog.c:854:llog_process_thread()) stop processing plain [0xa:0x2:0x0] index 7 count 7 00000020:01000000:1.0:1773941377.920728:0:6310:0:(obd_config.c:2178:class_config_parse_llog()) Processed log params gen 4-6 (rc=0) 10000000:01000000:1.0:1773941377.920746:0:6310:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'params' succeeded (0). 00010000:00010000:1.0:1773941377.920753:0:6310:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e284f lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xd03f0241a6d942be expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941377.920761:0:6310:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e284f lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xd03f0241a6d942be expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941378.171137:0:6308:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1773941379.195089:0:6781:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1773941380.219117:0:6307:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1773941381.243165:0:6308:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1773941382.267095:0:6781:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1773941382.395215:0:2403:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d111f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:2403:1860112291469696:192.168.204.114@tcp:400:kworker.0 00000100:00100000:2.0:1773941382.395215:0:2404:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d111f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:69e55c94-95bd-430e-b45e-3e120ab74187:2404:1860112291469568:192.168.204.114@tcp:400:kworker.0 00000100:00100000:2.0:1773941382.395343:0:2404:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d111f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:2404:1860112291469824:192.168.204.114@tcp:400:kworker.0 00000100:00100000:3.0:1773941382.395346:0:2403:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d111f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:2403:1860112291469952:192.168.204.114@tcp:400:kworker.0 00000100:00100000:2.0:1773941382.395374:0:2404:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d111f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:2404:1860112291470080:192.168.204.114@tcp:400:kworker.0 00000100:00100000:2.0:1773941382.396922:0:2404:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d111f2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:69e55c94-95bd-430e-b45e-3e120ab74187:2404:1860112291469568:192.168.204.114@tcp:400:kworker.0 00000100:00100000:3.0:1773941382.396930:0:2403:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d111f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:2403:1860112291469696:192.168.204.114@tcp:400:kworker.0 00010000:00080000:2.0:1773941382.396947:0:2404:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff988d111f0000 x1860112291469824/t0(0) o400->lustre-MDT0001-mdc-ffff988d07f1c800@192.168.204.114@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941398 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00080000:3.0:1773941382.396954:0:2403:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff988d111f3800 x1860112291469952/t0(0) o400->lustre-OST0000-osc-ffff988d07f1c800@192.168.204.114@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941398 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941382.396995:0:2404:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d111f0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:2404:1860112291469824:192.168.204.114@tcp:400:kworker.0 00000100:00100000:3.0:1773941382.397008:0:2403:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d111f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:2403:1860112291469952:192.168.204.114@tcp:400:kworker.0 00010000:00080000:2.0:1773941382.397016:0:2404:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff988d111f0e00 x1860112291470080/t0(0) o400->lustre-OST0001-osc-ffff988d07f1c800@192.168.204.114@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941398 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941382.397024:0:2404:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d111f0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:2404:1860112291470080:192.168.204.114@tcp:400:kworker.0 00010000:00010000:2.0:1773941383.291086:0:6307:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:3.0:1773941383.892484:0:7137:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg414-client.virtnet: executing check_logdir /tmp/testlogs/ 00010000:00010000:0.0:1773941384.315049:0:6308:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1773941385.341939:0:6781:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:3.0:1773941385.350610:0:7399:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg414-client.virtnet: executing yml_node 00010000:00010000:3.0:1773941386.364034:0:6307:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:0.0:1773941386.821773:0:7862:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: Client: 2.17.50.193 00010000:00010000:0.0:1773941387.387044:0:6308:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1773941387.515104:0:2403:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d111f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:69e55c94-95bd-430e-b45e-3e120ab74187:2403:1860112291470208:192.168.204.114@tcp:400:kworker.0 00000100:00100000:3.0:1773941387.515196:0:2403:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d111f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:2403:1860112291470464:192.168.204.114@tcp:400:kworker.0 00000100:00100000:3.0:1773941387.515235:0:2403:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d111f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:2403:1860112291470720:192.168.204.114@tcp:400:kworker.0 00000100:00100000:3.0:1773941387.515270:0:2403:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:3.0:1773941387.515276:0:2403:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d111f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:2403:1860112291470336:192.168.204.114@tcp:400:kworker.0 00000100:00100000:3.0:1773941387.515301:0:2403:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d111f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:2403:1860112291470592:192.168.204.114@tcp:400:kworker.0 00000100:00100000:3.0:1773941387.516339:0:2403:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d111f3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:69e55c94-95bd-430e-b45e-3e120ab74187:2403:1860112291470208:192.168.204.114@tcp:400:kworker.0 00010000:00080000:3.0:1773941387.516360:0:2403:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff988d111f3100 x1860112291470720/t0(0) o400->lustre-OST0001-osc-ffff988d07f1c800@192.168.204.114@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941403 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941387.516378:0:2403:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d111f3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:2403:1860112291470720:192.168.204.114@tcp:400:kworker.0 00000100:00100000:3.0:1773941387.516661:0:2403:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d111f2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:2403:1860112291470336:192.168.204.114@tcp:400:kworker.0 00010000:00080000:3.0:1773941387.516672:0:2403:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff988d111f1500 x1860112291470464/t0(0) o400->lustre-MDT0001-mdc-ffff988d07f1c800@192.168.204.114@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941403 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941387.516745:0:2403:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d111f1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:2403:1860112291470464:192.168.204.114@tcp:400:kworker.0 00010000:00080000:3.0:1773941387.516768:0:2403:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff988d111f0700 x1860112291470592/t0(0) o400->lustre-OST0000-osc-ffff988d07f1c800@192.168.204.114@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941403 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941387.516775:0:2403:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d111f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:2403:1860112291470592:192.168.204.114@tcp:400:kworker.0 00000001:02000400:0.0:1773941387.671209:0:8019:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: MDS: 2.17.50.193 00010000:00010000:2.0:1773941388.411067:0:6781:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:3.0:1773941388.458266:0:8175:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: OSS: 2.17.50.193 00000001:02000400:0.0:1773941388.931284:0:8322:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: -----============= acceptance-small: conf-sanity ============----- Thu Mar 19 13:29:48 EDT 2026 00010000:00010000:3.0:1773941389.436070:0:6307:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1773941390.459044:0:6308:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1773941391.483129:0:6781:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1773941392.507097:0:6307:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1773941392.635164:0:2402:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:69e55c94-95bd-430e-b45e-3e120ab74187:2402:1860112291470848:192.168.204.114@tcp:400:kworker.0 00000100:00100000:1.0:1773941392.635181:0:2401:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:2401:1860112291470976:192.168.204.114@tcp:400:kworker.0 00000100:00100000:1.0:1773941392.635275:0:2401:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:2401:1860112291471232:192.168.204.114@tcp:400:kworker.0 00000100:00100000:0.0:1773941392.635289:0:2402:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:2402:1860112291471104:192.168.204.114@tcp:400:kworker.0 00000100:00100000:0.0:1773941392.635326:0:2402:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:2402:1860112291471360:192.168.204.114@tcp:400:kworker.0 00010000:00080000:1.0:1773941392.636504:0:2402:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff988d203c3480 x1860112291471104/t0(0) o400->lustre-MDT0001-mdc-ffff988d07f1c800@192.168.204.114@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941408 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941392.636523:0:2402:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:2402:1860112291471104:192.168.204.114@tcp:400:kworker.0 00000100:00100000:0.0:1773941392.636525:0:2401:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:2401:1860112291470976:192.168.204.114@tcp:400:kworker.0 00010000:00080000:1.0:1773941392.636555:0:2402:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff988d203c2a00 x1860112291471360/t0(0) o400->lustre-OST0001-osc-ffff988d07f1c800@192.168.204.114@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941408 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00080000:0.0:1773941392.636558:0:2401:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=46162) req@ffff988d203c1180 x1860112291471232/t0(0) o400->lustre-OST0000-osc-ffff988d07f1c800@192.168.204.114@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941408 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941392.636580:0:2402:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:2402:1860112291471360:192.168.204.114@tcp:400:kworker.0 00000100:00100000:0.0:1773941392.636583:0:2401:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:900b97a3-8a4c-4ef5-9f63-261f870f4ac3:2401:1860112291471232:192.168.204.114@tcp:400:kworker.0 00000100:00100000:1.0:1773941392.636599:0:2402:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:69e55c94-95bd-430e-b45e-3e120ab74187:2402:1860112291470848:192.168.204.114@tcp:400:kworker.0 00010000:00010000:0.0:1773941393.531041:0:6308:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000001:02000400:3.0:1773941394.108663:0:8872:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: excepting tests: 32newtarball 108a 112a 112b 113 117 119 121 122a 123aa 123ab 123ac 123ad 123ae 123af 123ag 123ah 123ahi 123F 123G 123H 126 129 132 133 135 136 137 150 152 153a 153b 153c 155 802a 00000001:02000400:3.0:1773941394.539534:0:9011:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: skipping tests SLOW=no: 45 69 106 111 114 00010000:00010000:0.0:1773941394.555031:0:6781:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200008:1.0:1773941394.795950:0:9123:0:(file.c:6291:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff988d2052a1c8), request_mask 2047, flags 0, foreign 0 00000080:00200000:1.0:1773941394.795959:0:9123:0:(file.c:6186:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8),name=/ 00000002:00010000:1.0:1773941394.795993:0:9123:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00010000:00010000:1.0:1773941394.796009:0:9123:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff988d07f1c800 lock: ffff988d0168b000/0x8e86767622e283a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd03f0241a6d9429b expref: -99 pid: 6343 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00010000:00010000:1.0:1773941394.796021:0:9123:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff988d07f1c800 lock: ffff988d0168b000/0x8e86767622e283a lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd03f0241a6d9429b expref: -99 pid: 6343 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00000080:00200000:1.0:1773941394.796027:0:9123:0:(namei.c:951:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00010000:1.0:1773941394.796032:0:9123:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff988d2052a1c8) 00000080:00010000:1.0:1773941394.796035:0:9123:0:(llite_internal.h:1998:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff988d2052a1c8) for lock 0x8e86767622e283a 00000080:00010000:1.0:1773941394.796040:0:9123:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x8e86767622e283a from it ffffb6ec81d1fcb8 00010000:00010000:1.0:1773941394.796043:0:9123:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff988d07f1c800 lock: ffff988d0168b000/0x8e86767622e283a lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd03f0241a6d9429b expref: -99 pid: 6343 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00010000:00010000:1.0:1773941394.796049:0:9123:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff988d07f1c800 lock: ffff988d0168b000/0x8e86767622e283a lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd03f0241a6d9429b expref: -99 pid: 6343 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00000080:00200000:2.0:1773941394.967414:0:9124:0:(llite_lib.c:3557:ll_umount_begin()) VFS Op: superblock ffff988d07f1c800 count 1 active 9 00800000:00000080:2.0:1773941394.967423:0:9124:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d07f1c800: cmd=4240666a len=576 karg=ffff988d035a6c00 uarg=0000000000000000 00000002:00000080:2.0:1773941394.967431:0:9124:0:(mdc_request.c:2300:mdc_iocontrol()) lustre-MDT0000-mdc-ffff988d07f1c800: cmd=4240666a len=576 karg=ffff988d035a6c00 uarg=0000000000000000 00000100:02000400:2.0:1773941394.967435:0:9124:0:(recover.c:247:ptlrpc_set_import_active()) setting import lustre-MDT0000_UUID INACTIVE by administrator request 00000100:00080000:2.0:1773941394.969656:0:9124:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-MDT0000_UUID INVALID 00000100:00100000:2.0:1773941394.969661:0:9124:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff988d07f1c800: committing for last_committed 0 gen 2 00000100:00100000:2.0:1773941394.969669:0:9124:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00010000:00010000:2.0:1773941394.969780:0:9124:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff988d07f1c800 lock: ffff988d0168b000/0x8e86767622e283a lrc: 3/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x10e400000000 nid: local remote: 0xd03f0241a6d9429b expref: -99 pid: 6343 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00010000:00010000:2.0:1773941394.969799:0:9124:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:2.0:1773941394.969808:0:9124:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff988d07f1c800 lock: ffff988d0168b000/0x8e86767622e283a lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x490f400000000 nid: local remote: 0xd03f0241a6d9429b expref: -99 pid: 6343 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00000002:00000080:2.0:1773941394.971270:0:9124:0:(mdc_request.c:2300:mdc_iocontrol()) lustre-MDT0001-mdc-ffff988d07f1c800: cmd=4240666a len=576 karg=ffff988d035a6c00 uarg=0000000000000000 00000100:02000400:2.0:1773941394.971276:0:9124:0:(recover.c:247:ptlrpc_set_import_active()) setting import lustre-MDT0001_UUID INACTIVE by administrator request 00000100:00080000:2.0:1773941394.971278:0:9124:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-MDT0001_UUID INVALID 00000100:00100000:2.0:1773941394.971280:0:9124:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff988d07f1c800: committing for last_committed 0 gen 2 00000100:00100000:2.0:1773941394.971283:0:9124:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00020000:00000080:2.0:1773941394.972982:0:9124:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d07f1c800: cmd=4240666a len=576 karg=ffff988d035a6c00 uarg=0000000000000000 00000008:00000080:2.0:1773941394.972991:0:9124:0:(osc_request.c:3548:osc_iocontrol()) lustre-OST0000-osc-ffff988d07f1c800: cmd=4240666a len=576 karg=ffff988d035a6c00 uarg=0000000000000000 00000100:02000400:2.0:1773941394.972995:0:9124:0:(recover.c:247:ptlrpc_set_import_active()) setting import lustre-OST0000_UUID INACTIVE by administrator request 00020000:01000000:2.0:1773941394.973001:0:9124:0:(lov_obd.c:378:lov_set_osc_active()) deactivate OSC lustre-OST0000_UUID 00000080:00000004:2.0:1773941394.973013:0:9124:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:2.0:1773941394.973018:0:9124:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:2.0:1773941394.973023:0:9124:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-OST0000_UUID INVALID 00000100:00100000:2.0:1773941394.973025:0:9124:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0000-osc-ffff988d07f1c800: committing for last_committed 0 gen 2 00020000:01000000:2.0:1773941394.973029:0:9124:0:(lov_obd.c:389:lov_set_osc_active()) Marking OSC lustre-OST0000_UUID inactive 00000080:00000004:2.0:1773941394.973031:0:9124:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:2.0:1773941394.973033:0:9124:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00100000:2.0:1773941394.973036:0:9124:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000008:00000080:2.0:1773941394.973195:0:9124:0:(osc_request.c:3548:osc_iocontrol()) lustre-OST0001-osc-ffff988d07f1c800: cmd=4240666a len=576 karg=ffff988d035a6c00 uarg=0000000000000000 00000100:02000400:2.0:1773941394.973199:0:9124:0:(recover.c:247:ptlrpc_set_import_active()) setting import lustre-OST0001_UUID INACTIVE by administrator request 00020000:01000000:2.0:1773941394.973202:0:9124:0:(lov_obd.c:378:lov_set_osc_active()) deactivate OSC lustre-OST0001_UUID 00000080:00000004:2.0:1773941394.973204:0:9124:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:2.0:1773941394.973206:0:9124:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00080000:2.0:1773941394.973208:0:9124:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-OST0001_UUID INVALID 00000100:00100000:2.0:1773941394.973211:0:9124:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0001-osc-ffff988d07f1c800: committing for last_committed 0 gen 2 00020000:01000000:2.0:1773941394.973213:0:9124:0:(lov_obd.c:389:lov_set_osc_active()) Marking OSC lustre-OST0001_UUID inactive 00000080:00000004:2.0:1773941394.973215:0:9124:0:(lcommon_misc.c:94:cl_ocd_update()) Changing connect_flags: 0xa0425af2e3440038 -> 0xa0425af2e3440038 00000080:00080000:2.0:1773941394.973217:0:9124:0:(lcommon_misc.c:56:cl_init_ea_size()) updating def/max_easize: 72/65536 00000100:00100000:2.0:1773941394.973219:0:9124:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00010000:00010000:2.0:1773941395.579112:0:6307:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1773941396.603063:0:6308:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1773941397.627072:0:6781:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1773941397.755110:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 900b97a3-8a4c-4ef5-9f63-261f870f4ac3->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: FULL) 00000100:00080000:2.0:1773941397.755116:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 900b97a3-8a4c-4ef5-9f63-261f870f4ac3->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: FULL) 00000100:00080000:2.0:1773941397.755118:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 900b97a3-8a4c-4ef5-9f63-261f870f4ac3->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: FULL) 00000100:00080000:2.0:1773941397.755121:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 900b97a3-8a4c-4ef5-9f63-261f870f4ac3->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: FULL) 00000100:00100000:3.0:1773941397.755159:0:2403:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:3.0:1773941397.755179:0:2403:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06504000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:69e55c94-95bd-430e-b45e-3e120ab74187:2403:1860112291471488:192.168.204.114@tcp:400:kworker.0 00000100:00100000:3.0:1773941397.756431:0:2403:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06504000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:69e55c94-95bd-430e-b45e-3e120ab74187:2403:1860112291471488:192.168.204.114@tcp:400:kworker.0 00010000:00010000:2.0:1773941398.651101:0:6307:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1773941399.675082:0:6308:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1773941400.699161:0:6781:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1773941401.723183:0:6307:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1773941402.747123:0:6308:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1773941402.875260:0:461:0:(pinger.c:247:ptlrpc_pinger_process_import()) 900b97a3-8a4c-4ef5-9f63-261f870f4ac3->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: FULL) 00000100:00080000:1.0:1773941402.875278:0:461:0:(pinger.c:247:ptlrpc_pinger_process_import()) 900b97a3-8a4c-4ef5-9f63-261f870f4ac3->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: FULL) 00000100:00080000:1.0:1773941402.875300:0:461:0:(pinger.c:247:ptlrpc_pinger_process_import()) 900b97a3-8a4c-4ef5-9f63-261f870f4ac3->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: FULL) 00000100:00080000:1.0:1773941402.875304:0:461:0:(pinger.c:247:ptlrpc_pinger_process_import()) 900b97a3-8a4c-4ef5-9f63-261f870f4ac3->lustre-OST0001_UUID: not pinging (in recovery or recovery disabled: FULL) 00000100:00100000:0.0:1773941402.875306:0:2401:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:69e55c94-95bd-430e-b45e-3e120ab74187:2401:1860112291471616:192.168.204.114@tcp:400:kworker.0 00000100:00100000:1.0:1773941402.876703:0:2401:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:69e55c94-95bd-430e-b45e-3e120ab74187:2401:1860112291471616:192.168.204.114@tcp:400:kworker.0 00010000:00010000:0.0:1773941403.771145:0:6781:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1773941404.795161:0:6307:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1773941405.249059:0:9124:0:(llite_lib.c:1951:ll_clear_inode()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d2052a1c8) 00000080:00200000:2.0:1773941405.249085:0:9124:0:(llite_lib.c:1572:ll_put_super()) VFS Op: cfg_instance lustre-client-ffff988d07f1c800 (sb ffff988d07f1c800) 00010000:00010000:2.0:1773941405.249128:0:9124:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2841 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0xd03f0241a6d942a9 expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941405.249141:0:9124:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2841 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0xd03f0241a6d942a9 expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941405.249149:0:9124:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2841 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000400000000 nid: local remote: 0xd03f0241a6d942a9 expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941405.249157:0:9124:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2841 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000400000000 nid: local remote: 0xd03f0241a6d942a9 expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941405.249182:0:9124:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1773941405.249189:0:9124:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d421ff200/0x8e86767622e2802 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xd03f0241a6d941b4 expref: -99 pid: 6296 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941405.249194:0:9124:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d421ff200/0x8e86767622e2802 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0xd03f0241a6d941b4 expref: -99 pid: 6296 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941405.249198:0:9124:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d421ff200/0x8e86767622e2802 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000400000000 nid: local remote: 0xd03f0241a6d941b4 expref: -99 pid: 6296 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941405.249203:0:9124:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d421ff200/0x8e86767622e2802 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000400000000 nid: local remote: 0xd03f0241a6d941b4 expref: -99 pid: 6296 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941405.249208:0:6308:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1773941405.249217:0:9124:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00010000:2.0:1773941405.249223:0:9124:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e284f lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xd03f0241a6d942be expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941405.249228:0:9124:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e284f lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0xd03f0241a6d942be expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941405.249230:0:6308:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2841 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000400000000 nid: local remote: 0xd03f0241a6d942a9 expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941405.249247:0:6781:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1773941405.249250:0:9124:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e284f lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000400000000 nid: local remote: 0xd03f0241a6d942be expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941405.249267:0:6781:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d421ff200/0x8e86767622e2802 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000400000000 nid: local remote: 0xd03f0241a6d941b4 expref: -99 pid: 6296 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941405.249271:0:9124:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e284f lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000400000000 nid: local remote: 0xd03f0241a6d942be expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941405.249272:0:6308:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d01488e00 already unused, calling callback (ffffffffc12b4c20) 10000000:00010000:0.0:1773941405.249278:0:6308:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2841 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000400000000 nid: local remote: 0xd03f0241a6d942a9 expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941405.249284:0:6781:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d421ff200 already unused, calling callback (ffffffffc12b4c20) 10000000:00010000:1.0:1773941405.249290:0:6781:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d421ff200/0x8e86767622e2802 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000400000000 nid: local remote: 0xd03f0241a6d941b4 expref: -99 pid: 6296 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941405.249302:0:6308:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2841 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1008400000000 nid: local remote: 0xd03f0241a6d942a9 expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941405.249304:0:6781:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d421ff200/0x8e86767622e2802 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1008400000000 nid: local remote: 0xd03f0241a6d941b4 expref: -99 pid: 6296 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941405.249309:0:9124:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 10000000:01000000:2.0:1773941405.249319:0:9124:0:(mgc_request.c:547:config_log_end()) end config log lustre-client (0) 10000000:00010000:1.0:1773941405.249328:0:6781:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d421ff200/0x8e86767622e2802 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1009400000000 nid: local remote: 0xd03f0241a6d941b4 expref: -99 pid: 6296 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:0.0:1773941405.249330:0:6308:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2841 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1009400000000 nid: local remote: 0xd03f0241a6d942a9 expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941405.249338:0:6307:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 10000000:01000000:1.0:1773941405.249346:0:6781:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 10000000:01000000:0.0:1773941405.249350:0:6308:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:01000000:1.0:1773941405.249351:0:6781:0:(mgc_request.c:944:mgc_blocking_ast()) log lustre-cliir: stopping, won't requeue 10000000:01000000:1.0:1773941405.249354:0:6781:0:(mgc_request.c:130:config_log_put()) dropping config log lustre-cliir 10000000:01000000:0.0:1773941405.249357:0:6308:0:(mgc_request.c:944:mgc_blocking_ast()) log lustre-client: stopping, won't requeue 10000000:01000000:0.0:1773941405.249362:0:6308:0:(mgc_request.c:130:config_log_put()) dropping config log lustre-client 00010000:00010000:1.0:1773941405.249396:0:6781:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: MGC192.168.204.114@tcp lock: ffff988d421ff200/0x8e86767622e2802 lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5809400000020 nid: local remote: 0xd03f0241a6d941b4 expref: -99 pid: 6296 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941405.249396:0:6308:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2841 lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5809400000020 nid: local remote: 0xd03f0241a6d942a9 expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941405.249414:0:6308:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1773941405.249415:0:6781:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:1.0:1773941405.249441:0:6781:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d421ff200/0x8e86767622e2802 lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5809400000020 nid: local remote: 0xd03f0241a6d941b4 expref: -99 pid: 6296 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941405.249442:0:6308:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2841 lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5809400000020 nid: local remote: 0xd03f0241a6d942a9 expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941405.249458:0:6781:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d421ff200/0x8e86767622e2802 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5809400000020 nid: local remote: 0xd03f0241a6d941b4 expref: -99 pid: 6296 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941405.249460:0:6308:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2841 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5809400000020 nid: local remote: 0xd03f0241a6d942a9 expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941405.249484:0:6308:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1773941405.249488:0:6308:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1773941405.249492:0:6308:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1773941405.249511:0:2401:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10835880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:69e55c94-95bd-430e-b45e-3e120ab74187:2401:1860112291471744:192.168.204.114@tcp:103:ldlm_bl.0 00010000:00010000:1.0:1773941405.249512:0:6781:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1773941405.249518:0:6781:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1773941405.249522:0:9127:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1773941405.249525:0:6781:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1773941405.249534:0:6307:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e284f lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000400000000 nid: local remote: 0xd03f0241a6d942be expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941405.249548:0:6307:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d01488800 already unused, calling callback (ffffffffc12b4c20) 00000100:00100000:1.0:1773941405.249548:0:2402:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:69e55c94-95bd-430e-b45e-3e120ab74187:2402:1860112291471872:192.168.204.114@tcp:103:ldlm_bl.0 10000000:00010000:3.0:1773941405.249555:0:6307:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e284f lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000400000000 nid: local remote: 0xd03f0241a6d942be expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941405.249567:0:6307:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e284f lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1008400000000 nid: local remote: 0xd03f0241a6d942be expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:3.0:1773941405.249590:0:6307:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e284f lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1009400000000 nid: local remote: 0xd03f0241a6d942be expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1773941405.249597:0:6307:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x736d61726170:0x3:0x0].0x0 (params) 10000000:01000000:3.0:1773941405.249600:0:6307:0:(mgc_request.c:944:mgc_blocking_ast()) log params: stopping, won't requeue 10000000:01000000:3.0:1773941405.249603:0:6307:0:(mgc_request.c:130:config_log_put()) dropping config log params 00010000:00010000:3.0:1773941405.249627:0:6307:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e284f lrc: 2/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5809400000020 nid: local remote: 0xd03f0241a6d942be expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941405.249635:0:6307:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00010000:3.0:1773941405.249659:0:6307:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e284f lrc: 1/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5809400000020 nid: local remote: 0xd03f0241a6d942be expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941405.249669:0:6307:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e284f lrc: 0/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5809400000020 nid: local remote: 0xd03f0241a6d942be expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941405.249691:0:6307:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1773941405.249695:0:6307:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1773941405.249699:0:6307:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1773941405.249714:0:2403:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d111f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:69e55c94-95bd-430e-b45e-3e120ab74187:2403:1860112291472000:192.168.204.114@tcp:103:ldlm_bl.0 00010000:00010000:3.0:1773941405.249784:0:9127:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1773941405.249788:0:9127:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00020000:01000000:2.0:1773941405.249915:0:9124:0:(lov_obd.c:589:lov_del_target()) uuid: lustre-OST0000_UUID idx: 0 gen: 1 exp: ffff988d08358000 active: 0 00020000:01000000:2.0:1773941405.249919:0:9124:0:(lov_obd.c:589:lov_del_target()) uuid: lustre-OST0001_UUID idx: 1 gen: 1 exp: ffff988d08358800 active: 0 00020000:01000000:2.0:1773941405.249921:0:9124:0:(lov_obd.c:64:lov_tgts_putref()) destroying 2 lov targets 00020000:01000000:2.0:1773941405.249924:0:9124:0:(lov_obd.c:611:__lov_del_obd()) Removing tgt lustre-OST0001_UUID : lustre-OST0001-osc-ffff988d07f1c800 00020000:01000000:2.0:1773941405.249926:0:9124:0:(lov_obd.c:248:lov_disconnect_obd()) lustre-clilov-ffff988d07f1c800: disconnecting target lustre-OST0001-osc-ffff988d07f1c800 00000100:00080000:2.0:1773941405.249931:0:9124:0:(pinger.c:396:ptlrpc_pinger_del_import()) removing pingable import 900b97a3-8a4c-4ef5-9f63-261f870f4ac3->lustre-OST0001_UUID 00000100:00080000:2.0:1773941405.250050:0:9124:0:(import.c:65:import_set_state_nolock()) ffff988d20333800 lustre-OST0001_UUID: changing import state from FULL to CLOSED 00000100:00080000:2.0:1773941405.250057:0:9124:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-OST0001_UUID INVALID 00000100:00100000:2.0:1773941405.250060:0:9124:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0001-osc-ffff988d07f1c800: committing for last_committed 0 gen 3 00000100:00100000:2.0:1773941405.250064:0:9124:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000020:00000080:2.0:1773941405.250208:0:9124:0:(genops.c:1487:class_disconnect()) disconnect: cookie 0x8e86767622e2833 00000020:00000080:2.0:1773941405.250219:0:9124:0:(genops.c:904:class_export_put()) final put ffff988d08358800/LOV_OSC_UUID 00000020:01000000:2.0:1773941405.250233:0:9124:0:(obd_config.c:2502:class_manual_cleanup()) Manual cleanup of lustre-OST0001-osc-ffff988d07f1c800 (flags='F') 00000020:00000080:2.0:1773941405.250237:0:9124:0:(obd_config.c:1506:class_process_config()) processing cmd: cf004 00000020:00000080:2.0:1773941405.250243:0:9124:0:(obd_config.c:991:class_cleanup()) lustre-OST0001-osc-ffff988d07f1c800: forcing exports to disconnect: 0/3 00000020:00080000:2.0:1773941405.250246:0:9124:0:(genops.c:1577:class_disconnect_exports()) OBD device 6 (ffff988d203999b8) has no exports 00000020:00000080:1.0:1773941405.250252:0:461:0:(genops.c:863:class_export_destroy()) destroying export ffff988d08358800/LOV_OSC_UUID for lustre-OST0001-osc-ffff988d07f1c800 00000008:01000000:3.0:1773941405.250308:0:9124:0:(obd_class.h:663:obd_cleanup_client_import()) lustre-OST0001-osc-ffff988d07f1c800: client import never connected 00000100:00080000:3.0:1773941405.250312:0:9124:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-OST0001_UUID INVALID 00000100:00100000:3.0:1773941405.250315:0:9124:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0001-osc-ffff988d07f1c800: committing for last_committed 0 gen 4 00000100:00100000:3.0:1773941405.250319:0:9124:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000020:00000080:1.0:1773941405.250605:0:461:0:(genops.c:1202:obd_zombie_import_free()) destroying import ffff988d20333800 for lustre-OST0001-osc-ffff988d07f1c800 00000100:00100000:1.0:1773941405.250853:0:2402:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:69e55c94-95bd-430e-b45e-3e120ab74187:2402:1860112291471872:192.168.204.114@tcp:103:ldlm_bl.0 00000100:00100000:0.0:1773941405.250877:0:2401:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10835880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:69e55c94-95bd-430e-b45e-3e120ab74187:2401:1860112291471744:192.168.204.114@tcp:103:ldlm_bl.0 00000100:00100000:2.0:1773941405.250921:0:2403:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d111f0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:69e55c94-95bd-430e-b45e-3e120ab74187:2403:1860112291472000:192.168.204.114@tcp:103:ldlm_bl.0 00000020:00000080:3.0:1773941405.251286:0:9124:0:(obd_config.c:1506:class_process_config()) processing cmd: cf002 00000020:00000080:3.0:1773941405.251294:0:9124:0:(obd_config.c:919:class_detach()) detach on obd lustre-OST0001-osc-ffff988d07f1c800 (uuid 900b97a3-8a4c-4ef5-9f63-261f870f4ac3) 00000020:00000080:3.0:1773941405.251297:0:9124:0:(genops.c:904:class_export_put()) final put ffff988d20332800/900b97a3-8a4c-4ef5-9f63-261f870f4ac3 00000020:00000080:3.0:1773941405.251307:0:9124:0:(genops.c:863:class_export_destroy()) destroying export ffff988d20332800/900b97a3-8a4c-4ef5-9f63-261f870f4ac3 for lustre-OST0001-osc-ffff988d07f1c800 00000020:01000000:3.0:1773941405.251313:0:9124:0:(genops.c:440:class_free_dev()) finishing cleanup of obd lustre-OST0001-osc-ffff988d07f1c800 (900b97a3-8a4c-4ef5-9f63-261f870f4ac3) 00020000:01000000:3.0:1773941405.251498:0:9124:0:(lov_obd.c:611:__lov_del_obd()) Removing tgt lustre-OST0000_UUID : lustre-OST0000-osc-ffff988d07f1c800 00020000:01000000:3.0:1773941405.251501:0:9124:0:(lov_obd.c:248:lov_disconnect_obd()) lustre-clilov-ffff988d07f1c800: disconnecting target lustre-OST0000-osc-ffff988d07f1c800 00000100:00080000:3.0:1773941405.251505:0:9124:0:(pinger.c:396:ptlrpc_pinger_del_import()) removing pingable import 900b97a3-8a4c-4ef5-9f63-261f870f4ac3->lustre-OST0000_UUID 00000100:00080000:3.0:1773941405.251603:0:9124:0:(import.c:65:import_set_state_nolock()) ffff988d20331800 lustre-OST0000_UUID: changing import state from FULL to CLOSED 00000100:00080000:3.0:1773941405.251608:0:9124:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-OST0000_UUID INVALID 00000100:00100000:3.0:1773941405.251611:0:9124:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0000-osc-ffff988d07f1c800: committing for last_committed 0 gen 3 00000100:00100000:3.0:1773941405.251615:0:9124:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000020:00000080:3.0:1773941405.251769:0:9124:0:(genops.c:1487:class_disconnect()) disconnect: cookie 0x8e86767622e282c 00000020:00000080:3.0:1773941405.251782:0:9124:0:(genops.c:904:class_export_put()) final put ffff988d08358000/LOV_OSC_UUID 00000020:01000000:3.0:1773941405.251796:0:9124:0:(obd_config.c:2502:class_manual_cleanup()) Manual cleanup of lustre-OST0000-osc-ffff988d07f1c800 (flags='F') 00000020:00000080:3.0:1773941405.251800:0:9124:0:(obd_config.c:1506:class_process_config()) processing cmd: cf004 00000020:00000080:3.0:1773941405.251805:0:9124:0:(obd_config.c:991:class_cleanup()) lustre-OST0000-osc-ffff988d07f1c800: forcing exports to disconnect: 0/3 00000020:00080000:3.0:1773941405.251809:0:9124:0:(genops.c:1577:class_disconnect_exports()) OBD device 5 (ffff988d20398000) has no exports 00000020:00000080:1.0:1773941405.251813:0:461:0:(genops.c:863:class_export_destroy()) destroying export ffff988d08358000/LOV_OSC_UUID for lustre-OST0000-osc-ffff988d07f1c800 00000008:01000000:3.0:1773941405.251824:0:9124:0:(obd_class.h:663:obd_cleanup_client_import()) lustre-OST0000-osc-ffff988d07f1c800: client import never connected 00000100:00080000:3.0:1773941405.251828:0:9124:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-OST0000_UUID INVALID 00000100:00100000:3.0:1773941405.251831:0:9124:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0000-osc-ffff988d07f1c800: committing for last_committed 0 gen 4 00000100:00100000:3.0:1773941405.251834:0:9124:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000020:00000080:1.0:1773941405.252058:0:461:0:(genops.c:1202:obd_zombie_import_free()) destroying import ffff988d20331800 for lustre-OST0000-osc-ffff988d07f1c800 00000020:00000080:3.0:1773941405.252258:0:9124:0:(obd_config.c:1506:class_process_config()) processing cmd: cf002 00000020:00000080:3.0:1773941405.252264:0:9124:0:(obd_config.c:919:class_detach()) detach on obd lustre-OST0000-osc-ffff988d07f1c800 (uuid 900b97a3-8a4c-4ef5-9f63-261f870f4ac3) 00000020:00000080:3.0:1773941405.252267:0:9124:0:(genops.c:904:class_export_put()) final put ffff988d20331000/900b97a3-8a4c-4ef5-9f63-261f870f4ac3 00000020:00000080:3.0:1773941405.252270:0:9124:0:(genops.c:863:class_export_destroy()) destroying export ffff988d20331000/900b97a3-8a4c-4ef5-9f63-261f870f4ac3 for lustre-OST0000-osc-ffff988d07f1c800 00000020:01000000:3.0:1773941405.252274:0:9124:0:(genops.c:440:class_free_dev()) finishing cleanup of obd lustre-OST0000-osc-ffff988d07f1c800 (900b97a3-8a4c-4ef5-9f63-261f870f4ac3) 00000020:00000080:3.0:1773941405.252544:0:9124:0:(genops.c:1487:class_disconnect()) disconnect: cookie 0x8e86767622e2825 00000020:00000080:3.0:1773941405.252550:0:9124:0:(genops.c:904:class_export_put()) final put ffff988d0835d000/900b97a3-8a4c-4ef5-9f63-261f870f4ac3 00000020:00000080:1.0:1773941405.252581:0:461:0:(genops.c:863:class_export_destroy()) destroying export ffff988d0835d000/900b97a3-8a4c-4ef5-9f63-261f870f4ac3 for lustre-clilov-ffff988d07f1c800 00000100:00080000:3.0:1773941405.252887:0:9124:0:(pinger.c:396:ptlrpc_pinger_del_import()) removing pingable import 900b97a3-8a4c-4ef5-9f63-261f870f4ac3->lustre-MDT0000_UUID 00000100:00080000:3.0:1773941405.254272:0:9124:0:(import.c:65:import_set_state_nolock()) ffff988d20336800 lustre-MDT0000_UUID: changing import state from FULL to CLOSED 00000100:00080000:3.0:1773941405.254286:0:9124:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-MDT0000_UUID INVALID 00000100:00100000:3.0:1773941405.254289:0:9124:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff988d07f1c800: committing for last_committed 0 gen 3 00000100:00100000:3.0:1773941405.254293:0:9124:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000020:00000080:3.0:1773941405.256490:0:9124:0:(genops.c:1487:class_disconnect()) disconnect: cookie 0x8e86767622e2817 00000020:00000080:3.0:1773941405.256498:0:9124:0:(genops.c:904:class_export_put()) final put ffff988d08f0b000/900b97a3-8a4c-4ef5-9f63-261f870f4ac3 00000020:00000080:1.0:1773941405.256550:0:461:0:(genops.c:863:class_export_destroy()) destroying export ffff988d08f0b000/900b97a3-8a4c-4ef5-9f63-261f870f4ac3 for lustre-MDT0000-mdc-ffff988d07f1c800 00000100:00080000:3.0:1773941405.256592:0:9124:0:(pinger.c:396:ptlrpc_pinger_del_import()) removing pingable import 900b97a3-8a4c-4ef5-9f63-261f870f4ac3->lustre-MDT0001_UUID 00000100:00080000:3.0:1773941405.257919:0:9124:0:(import.c:65:import_set_state_nolock()) ffff988d20330800 lustre-MDT0001_UUID: changing import state from FULL to CLOSED 00000100:00080000:3.0:1773941405.257926:0:9124:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-MDT0001_UUID INVALID 00000100:00100000:3.0:1773941405.257929:0:9124:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff988d07f1c800: committing for last_committed 0 gen 3 00000100:00100000:3.0:1773941405.257933:0:9124:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000020:00000080:3.0:1773941405.260171:0:9124:0:(genops.c:1487:class_disconnect()) disconnect: cookie 0x8e86767622e281e 00000020:00000080:3.0:1773941405.260179:0:9124:0:(genops.c:904:class_export_put()) final put ffff988d08f0c800/900b97a3-8a4c-4ef5-9f63-261f870f4ac3 00000020:00000080:3.0:1773941405.260220:0:9124:0:(genops.c:1487:class_disconnect()) disconnect: cookie 0x8e86767622e2810 00000020:00000080:3.0:1773941405.260225:0:9124:0:(genops.c:904:class_export_put()) final put ffff988d08f08000/900b97a3-8a4c-4ef5-9f63-261f870f4ac3 00000020:00000080:1.0:1773941405.260231:0:461:0:(genops.c:863:class_export_destroy()) destroying export ffff988d08f0c800/900b97a3-8a4c-4ef5-9f63-261f870f4ac3 for lustre-MDT0001-mdc-ffff988d07f1c800 00000020:00000080:1.0:1773941405.260239:0:461:0:(genops.c:863:class_export_destroy()) destroying export ffff988d08f08000/900b97a3-8a4c-4ef5-9f63-261f870f4ac3 for lustre-clilmv-ffff988d07f1c800 00000020:01000000:3.0:1773941405.260243:0:9124:0:(obd_config.c:2502:class_manual_cleanup()) Manual cleanup of lustre-clilov-ffff988d07f1c800 (flags='F') 00000020:00000080:3.0:1773941405.260250:0:9124:0:(obd_config.c:1506:class_process_config()) processing cmd: cf004 00000020:00000080:3.0:1773941405.260256:0:9124:0:(obd_config.c:991:class_cleanup()) lustre-clilov-ffff988d07f1c800: forcing exports to disconnect: 0/1 00000020:00080000:3.0:1773941405.260260:0:9124:0:(genops.c:1577:class_disconnect_exports()) OBD device 1 (ffff988d089acd28) has no exports 00000020:00000080:3.0:1773941405.260287:0:9124:0:(obd_config.c:1506:class_process_config()) processing cmd: cf002 00000020:00000080:3.0:1773941405.260293:0:9124:0:(obd_config.c:919:class_detach()) detach on obd lustre-clilov-ffff988d07f1c800 (uuid 900b97a3-8a4c-4ef5-9f63-261f870f4ac3) 00000020:00000080:3.0:1773941405.260296:0:9124:0:(genops.c:904:class_export_put()) final put ffff988d035fa000/900b97a3-8a4c-4ef5-9f63-261f870f4ac3 00000020:00000080:3.0:1773941405.260299:0:9124:0:(genops.c:863:class_export_destroy()) destroying export ffff988d035fa000/900b97a3-8a4c-4ef5-9f63-261f870f4ac3 for lustre-clilov-ffff988d07f1c800 00000020:01000000:3.0:1773941405.260305:0:9124:0:(genops.c:440:class_free_dev()) finishing cleanup of obd lustre-clilov-ffff988d07f1c800 (900b97a3-8a4c-4ef5-9f63-261f870f4ac3) 00000020:01000000:3.0:1773941405.260313:0:9124:0:(obd_config.c:2502:class_manual_cleanup()) Manual cleanup of lustre-clilmv-ffff988d07f1c800 (flags='F') 00000020:00000080:3.0:1773941405.260316:0:9124:0:(obd_config.c:1506:class_process_config()) processing cmd: cf004 00000020:00000080:3.0:1773941405.260320:0:9124:0:(obd_config.c:991:class_cleanup()) lustre-clilmv-ffff988d07f1c800: forcing exports to disconnect: 0/1 00000020:00080000:3.0:1773941405.260322:0:9124:0:(genops.c:1577:class_disconnect_exports()) OBD device 2 (ffff988d089ab370) has no exports 00000020:00000080:3.0:1773941405.260361:0:9124:0:(obd_config.c:1506:class_process_config()) processing cmd: cf002 00000020:00000080:3.0:1773941405.260364:0:9124:0:(obd_config.c:919:class_detach()) detach on obd lustre-clilmv-ffff988d07f1c800 (uuid 900b97a3-8a4c-4ef5-9f63-261f870f4ac3) 00000020:00000080:3.0:1773941405.260375:0:9124:0:(genops.c:904:class_export_put()) final put ffff988d20334800/900b97a3-8a4c-4ef5-9f63-261f870f4ac3 00000020:00000080:3.0:1773941405.260378:0:9124:0:(genops.c:863:class_export_destroy()) destroying export ffff988d20334800/900b97a3-8a4c-4ef5-9f63-261f870f4ac3 for lustre-clilmv-ffff988d07f1c800 00000020:01000000:3.0:1773941405.260382:0:9124:0:(genops.c:440:class_free_dev()) finishing cleanup of obd lustre-clilmv-ffff988d07f1c800 (900b97a3-8a4c-4ef5-9f63-261f870f4ac3) 00000020:01000000:3.0:1773941405.260525:0:9124:0:(obd_config.c:2502:class_manual_cleanup()) Manual cleanup of lustre-MDT0000-mdc-ffff988d07f1c800 (flags='F') 00000020:00000080:3.0:1773941405.260529:0:9124:0:(obd_config.c:1506:class_process_config()) processing cmd: cf004 00000020:00000080:3.0:1773941405.260532:0:9124:0:(obd_config.c:991:class_cleanup()) lustre-MDT0000-mdc-ffff988d07f1c800: forcing exports to disconnect: 0/2 00000020:00080000:3.0:1773941405.260534:0:9124:0:(genops.c:1577:class_disconnect_exports()) OBD device 3 (ffff988d089a8000) has no exports 00000008:01000000:3.0:1773941405.260545:0:9124:0:(obd_class.h:663:obd_cleanup_client_import()) lustre-MDT0000-mdc-ffff988d07f1c800: client import never connected 00000100:00080000:3.0:1773941405.260548:0:9124:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-MDT0000_UUID INVALID 00000100:00100000:3.0:1773941405.260551:0:9124:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff988d07f1c800: committing for last_committed 0 gen 4 00000100:00100000:3.0:1773941405.260555:0:9124:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000020:00000080:0.0:1773941405.263087:0:36:0:(genops.c:1202:obd_zombie_import_free()) destroying import ffff988d20336800 for lustre-MDT0000-mdc-ffff988d07f1c800 00000020:00000080:3.0:1773941405.263327:0:9124:0:(obd_config.c:1506:class_process_config()) processing cmd: cf002 00000020:00000080:3.0:1773941405.263332:0:9124:0:(obd_config.c:919:class_detach()) detach on obd lustre-MDT0000-mdc-ffff988d07f1c800 (uuid 900b97a3-8a4c-4ef5-9f63-261f870f4ac3) 00000020:00000080:3.0:1773941405.263335:0:9124:0:(genops.c:904:class_export_put()) final put ffff988d20335800/900b97a3-8a4c-4ef5-9f63-261f870f4ac3 00000020:00000080:3.0:1773941405.263338:0:9124:0:(genops.c:863:class_export_destroy()) destroying export ffff988d20335800/900b97a3-8a4c-4ef5-9f63-261f870f4ac3 for lustre-MDT0000-mdc-ffff988d07f1c800 00000020:01000000:3.0:1773941405.263341:0:9124:0:(genops.c:440:class_free_dev()) finishing cleanup of obd lustre-MDT0000-mdc-ffff988d07f1c800 (900b97a3-8a4c-4ef5-9f63-261f870f4ac3) 00000020:01000000:0.0:1773941405.265075:0:9124:0:(obd_config.c:2502:class_manual_cleanup()) Manual cleanup of lustre-MDT0001-mdc-ffff988d07f1c800 (flags='F') 00000020:00000080:0.0:1773941405.265085:0:9124:0:(obd_config.c:1506:class_process_config()) processing cmd: cf004 00000020:00000080:0.0:1773941405.265094:0:9124:0:(obd_config.c:991:class_cleanup()) lustre-MDT0001-mdc-ffff988d07f1c800: forcing exports to disconnect: 0/2 00000020:00080000:0.0:1773941405.265098:0:9124:0:(genops.c:1577:class_disconnect_exports()) OBD device 4 (ffff988d2039b370) has no exports 00000008:01000000:0.0:1773941405.265113:0:9124:0:(obd_class.h:663:obd_cleanup_client_import()) lustre-MDT0001-mdc-ffff988d07f1c800: client import never connected 00000100:00080000:0.0:1773941405.265117:0:9124:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-MDT0001_UUID INVALID 00000100:00100000:0.0:1773941405.265121:0:9124:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff988d07f1c800: committing for last_committed 0 gen 4 00000100:00100000:0.0:1773941405.265126:0:9124:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000020:00000080:3.0:1773941405.267426:0:36:0:(genops.c:1202:obd_zombie_import_free()) destroying import ffff988d20330800 for lustre-MDT0001-mdc-ffff988d07f1c800 00000020:00000080:0.0:1773941405.267818:0:9124:0:(obd_config.c:1506:class_process_config()) processing cmd: cf002 00000020:00000080:0.0:1773941405.267825:0:9124:0:(obd_config.c:919:class_detach()) detach on obd lustre-MDT0001-mdc-ffff988d07f1c800 (uuid 900b97a3-8a4c-4ef5-9f63-261f870f4ac3) 00000020:00000080:0.0:1773941405.267829:0:9124:0:(genops.c:904:class_export_put()) final put ffff988d20330000/900b97a3-8a4c-4ef5-9f63-261f870f4ac3 00000020:00000080:0.0:1773941405.267833:0:9124:0:(genops.c:863:class_export_destroy()) destroying export ffff988d20330000/900b97a3-8a4c-4ef5-9f63-261f870f4ac3 for lustre-MDT0001-mdc-ffff988d07f1c800 00000020:01000000:0.0:1773941405.267837:0:9124:0:(genops.c:440:class_free_dev()) finishing cleanup of obd lustre-MDT0001-mdc-ffff988d07f1c800 (900b97a3-8a4c-4ef5-9f63-261f870f4ac3) 00000080:02000400:1.0:1773941405.269055:0:9124:0:(llite_lib.c:1617:ll_put_super()) Unmounted lustre-client 00000020:01000000:1.0:1773941405.270456:0:9124:0:(obd_config.c:1233:class_del_profile()) Del profile lustre-client 00000020:01000004:1.0:1773941405.270524:0:9124:0:(obd_mount.c:911:lustre_common_put_super()) dropping sb ffff988d07f1c800 00000100:00080000:1.0:1773941405.270538:0:9124:0:(pinger.c:396:ptlrpc_pinger_del_import()) removing pingable import 69e55c94-95bd-430e-b45e-3e120ab74187->MGS 00010000:00010000:1.0:1773941405.270555:0:9124:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d01488000/0x8e86767622e2848 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x1008400000000 nid: local remote: 0xd03f0241a6d942b0 expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:1.0:1773941405.270567:0:9124:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d01488000/0x8e86767622e2848 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x1009400000000 nid: local remote: 0xd03f0241a6d942b0 expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1773941405.270573:0:9124:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:01000000:1.0:1773941405.270576:0:9124:0:(mgc_request.c:954:mgc_blocking_ast()) log lustre-s: disconnecting, won't requeue 10000000:01000000:1.0:1773941405.270585:0:9124:0:(mgc_request.c:130:config_log_put()) dropping config log lustre-sptlrpc 00010000:00010000:1.0:1773941405.270630:0:9124:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: MGC192.168.204.114@tcp lock: ffff988d01488000/0x8e86767622e2848 lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x5809400000020 nid: local remote: 0xd03f0241a6d942b0 expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941405.270638:0:9124:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00000100:00100000:1.0:1773941405.270662:0:9124:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1056cf00 going to sleep for 0 seconds 00000100:00100000:1.0:1773941405.270676:0:9124:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0380 pname:cluuid:pid:xid:nid:opc:job umount:69e55c94-95bd-430e-b45e-3e120ab74187:9124:1860112291472128:192.168.204.114@tcp:103:umount.0 00010000:00010000:1.0:1773941405.270690:0:9124:0:(ldlm_request.c:1494:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00010000:00010000:1.0:1773941405.270703:0:9124:0:(ldlm_request.c:2277:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00010000:1.0:1773941405.270706:0:9124:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d01488000/0x8e86767622e2848 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x5809400000020 nid: local remote: 0xd03f0241a6d942b0 expref: -99 pid: 6310 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00080000:1.0:1773941405.270737:0:9124:0:(import.c:65:import_set_state_nolock()) ffff988d05486000 MGS: changing import state from FULL to CONNECTING 00000100:00100000:2.0:1773941405.270773:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:69e55c94-95bd-430e-b45e-3e120ab74187:2400:1860112291472256:192.168.204.114@tcp:251:umount.0 00000100:00080000:2.0:1773941405.272242:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d05486000 MGS: changing import state from CONNECTING to CLOSED 10000000:01000000:2.0:1773941405.272246:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941405.272248:0:2400:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:01000000:2.0:1773941405.272249:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 00000100:00100000:2.0:1773941405.272258:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:69e55c94-95bd-430e-b45e-3e120ab74187:2400:1860112291472256:192.168.204.114@tcp:251:umount.0 00000100:00080000:1.0:1773941405.272292:0:9124:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 10000000:01000000:1.0:1773941405.272295:0:9124:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 00000100:00100000:1.0:1773941405.272298:0:9124:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 10000000:01000000:1.0:1773941405.272300:0:9124:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000020:00000080:1.0:1773941405.272306:0:9124:0:(genops.c:1487:class_disconnect()) disconnect: cookie 0x8e86767622e27c3 00000020:00000080:1.0:1773941405.272313:0:9124:0:(genops.c:904:class_export_put()) final put ffff988d1977c000/69e55c94-95bd-430e-b45e-3e120ab74187 00000020:01000000:1.0:1773941405.272327:0:9124:0:(obd_config.c:2502:class_manual_cleanup()) Manual cleanup of MGC192.168.204.114@tcp (flags='') 00000020:00000080:1.0:1773941405.272332:0:9124:0:(obd_config.c:1506:class_process_config()) processing cmd: cf004 00000020:00000080:1.0:1773941405.272338:0:9124:0:(obd_config.c:991:class_cleanup()) MGC192.168.204.114@tcp: forcing exports to disconnect: 0/3 00000020:00080000:1.0:1773941405.272343:0:9124:0:(genops.c:1577:class_disconnect_exports()) OBD device 0 (ffff988d089a99b8) has no exports 00000020:00000080:3.0:1773941405.272346:0:36:0:(genops.c:863:class_export_destroy()) destroying export ffff988d1977c000/69e55c94-95bd-430e-b45e-3e120ab74187 for MGC192.168.204.114@tcp 10000000:01000000:0.0:1773941405.272393:0:6310:0:(mgc_request.c:708:mgc_requeue_thread()) Ending requeue thread 10000000:01000000:1.0:1773941405.272404:0:9124:0:(obd_class.h:663:obd_cleanup_client_import()) MGC192.168.204.114@tcp: client import never connected 00000100:00080000:1.0:1773941405.272407:0:9124:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 10000000:01000000:1.0:1773941405.272409:0:9124:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 00000100:00100000:1.0:1773941405.272411:0:9124:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 10000000:01000000:1.0:1773941405.272413:0:9124:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000020:00000080:3.0:1773941405.272450:0:36:0:(genops.c:1202:obd_zombie_import_free()) destroying import ffff988d05486000 for MGC192.168.204.114@tcp 00000020:00000080:1.0:1773941405.272478:0:9124:0:(obd_config.c:1506:class_process_config()) processing cmd: cf002 00000020:00000080:1.0:1773941405.272484:0:9124:0:(obd_config.c:919:class_detach()) detach on obd MGC192.168.204.114@tcp (uuid 69e55c94-95bd-430e-b45e-3e120ab74187) 00000020:00000080:1.0:1773941405.272487:0:9124:0:(genops.c:904:class_export_put()) final put ffff988d07f1d800/69e55c94-95bd-430e-b45e-3e120ab74187 00000020:00000080:1.0:1773941405.272490:0:9124:0:(genops.c:863:class_export_destroy()) destroying export ffff988d07f1d800/69e55c94-95bd-430e-b45e-3e120ab74187 for MGC192.168.204.114@tcp 00000020:01000000:1.0:1773941405.272502:0:9124:0:(genops.c:440:class_free_dev()) finishing cleanup of obd MGC192.168.204.114@tcp (69e55c94-95bd-430e-b45e-3e120ab74187) 00010000:00010000:0.0:1773941405.272691:0:6308:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (-1 blwis in pool) 00010000:00010000:2.0:1773941405.272752:0:6781:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (-2 blwis in pool) 00010000:00010000:3.0:1773941405.272804:0:6307:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (-3 blwis in pool) 00010000:00010000:0.0:1773941405.272848:0:9127:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (-4 blwis in pool) 00000100:00100000:1.0:1773941405.272942:0:6298:0:(service.c:3162:ptlrpc_main()) ldlm_cb00_001: service thread [ffff988d03111800:6298] 1 exiting: rc = 0 00000100:00100000:1.0:1773941405.273099:0:6297:0:(service.c:3162:ptlrpc_main()) ldlm_cb00_000: service thread [ffff988d0a32fa00:6297] 0 exiting: rc = 1 00000100:00100000:2.0:1773941405.273334:0:6299:0:(service.c:3162:ptlrpc_main()) ldlm_cb01_000: service thread [ffff988d03110000:6299] 0 exiting: rc = 1 00000100:00100000:3.0:1773941405.273347:0:6300:0:(service.c:3162:ptlrpc_main()) ldlm_cb01_001: service thread [ffff988d19734000:6300] 1 exiting: rc = 0 00000100:00100000:0.0:1773941405.273399:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:0.0:1773941405.273412:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:0.0:1773941405.273762:0:6302:0:(service.c:3162:ptlrpc_main()) ldlm_cn00_001: service thread [ffff988d02d18400:6302] 1 exiting: rc = 0 00000100:00100000:1.0:1773941405.273764:0:6301:0:(service.c:3162:ptlrpc_main()) ldlm_cn00_000: service thread [ffff988d02d18000:6301] 0 exiting: rc = 1 00000100:00100000:1.0:1773941405.273849:0:6303:0:(service.c:3162:ptlrpc_main()) ldlm_cn00_002: service thread [ffff988d02d18800:6303] 2 exiting: rc = 0 00000100:00100000:2.0:1773941405.274037:0:6304:0:(service.c:3162:ptlrpc_main()) ldlm_cn01_000: service thread [ffff988d1a004200:6304] 0 exiting: rc = 1 00000100:00100000:3.0:1773941405.274051:0:6305:0:(service.c:3162:ptlrpc_main()) ldlm_cn01_001: service thread [ffff988d0627ae00:6305] 1 exiting: rc = 0 00000100:00100000:2.0:1773941405.274138:0:6306:0:(service.c:3162:ptlrpc_main()) ldlm_cn01_002: service thread [ffff988d0627b000:6306] 2 exiting: rc = 0 00000100:00100000:1.0:1773941405.274221:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274235:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274248:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274272:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274287:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274297:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274307:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274316:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274324:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274335:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274345:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274355:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274365:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274374:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274384:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274394:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274404:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274413:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274421:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274429:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274432:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274435:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274438:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274442:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274445:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274449:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274452:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274455:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274466:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274469:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274473:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274477:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274480:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274483:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274487:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274490:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274494:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274497:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274500:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274503:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274507:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274510:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274514:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274517:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274520:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274523:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274526:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274529:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274533:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274536:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274540:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274544:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274548:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274551:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274555:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274559:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274563:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274566:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274570:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274574:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274578:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274581:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274584:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274591:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274596:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274600:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274604:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274608:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274612:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274616:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274619:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274623:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274627:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274631:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274634:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274637:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274641:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274644:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274647:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274651:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274654:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274657:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274660:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274663:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274666:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274670:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274673:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274677:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274681:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274684:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274687:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274691:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274694:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274697:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274701:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274704:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274707:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274710:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274717:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274721:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274725:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274729:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274732:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274735:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274739:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274742:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274745:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274748:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274751:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274754:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274757:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274761:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274764:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274768:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274771:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274775:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274778:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274782:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274785:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274789:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274793:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274796:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274799:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274803:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274806:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274809:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274812:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00100000:1.0:1773941405.274815:0:9124:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000020:01000004:1.0:1773941405.275332:0:9124:0:(obd_mount.c:724:lustre_put_lsi()) put ffff988d07f1c800 1 00000020:01000004:1.0:1773941405.275336:0:9124:0:(obd_mount.c:682:lustre_free_lsi()) Freeing lsi ffff988d07f1f800 00000001:02000400:0.0:1773941457.005379:0:11957:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg414-client.virtnet: executing wait_import_state_mount FULL mdc.lustre-MDT0000-mdc-*.mds_server_uuid 00000001:02000400:0.0:1773941458.165002:0:12202:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg414-client.virtnet: executing wait_import_state_mount FULL mdc.lustre-MDT0001-mdc-*.mds_server_uuid 00000001:02000400:2.0:1773941465.123910:0:12634:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg414-client.virtnet: executing wait_import_state_mount (FULL|IDLE) osc.lustre-OST0000-osc-[-0-9a-f]*.ost_server_uuid 00000001:02000400:3.0:1773941481.681775:0:13306:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: == conf-sanity test 161b: test mount with multiple IP for a single hostname ========================================================== 13:31:21 (1773941481) 00000001:02000400:3.0:1773941490.370128:0:13880:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg414-client.virtnet: executing wait_import_state_mount FULL mdc.lustre-MDT0000-mdc-*.mds_server_uuid 00000001:02000400:1.0:1773941491.622245:0:14125:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg414-client.virtnet: executing wait_import_state_mount FULL mdc.lustre-MDT0001-mdc-*.mds_server_uuid 00000001:02000400:3.0:1773941497.186080:0:14549:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: oleg414-client.virtnet: executing wait_import_state_mount (FULL|IDLE) osc.lustre-OST0000-osc-[-0-9a-f]*.ost_server_uuid 00000080:00000010:1.0:1773941497.289022:0:14577:0:(super25.c:253:lustre_init_fs_context()) kmalloced '(lmd)': 128 at ffff988d0716b880. 00000020:00000010:1.0:1773941497.289037:0:14577:0:(obd_mount.c:1388:lustre_parse_monolithic()) kmalloced '(lmd->lmd_params)': 4096 at ffff988d035d4000. 00000020:00000010:1.0:1773941497.289056:0:14577:0:(obd_mount.c:1590:lustre_parse_monolithic()) kmalloced '(lmd->lmd_dev)': 28 at ffff988d0652fec0. 00000020:00000010:1.0:1773941497.289060:0:14577:0:(obd_mount.c:1646:lustre_parse_monolithic()) kmalloced '(lmd->lmd_profile)': 14 at ffff988d02eebe30. 00000020:00000010:1.0:1773941497.289066:0:14577:0:(obd_mount.c:1674:lustre_parse_monolithic()) kmalloced '(lmd->lmd_opts)': 17 at ffff988d0652f600. 00000020:00000010:1.0:1773941497.289191:0:14577:0:(obd_mount.c:656:lustre_init_lsi()) kmalloced '(lsi)': 1880 at ffff988d08f0a800. 00000020:00000010:1.0:1773941497.289211:0:14577:0:(obd_mount.c:345:lustre_start_mgc()) kmalloced '(mgcname)': 23 at ffff988d0300eee0. 00000020:00000010:1.0:1773941497.289215:0:14577:0:(obd_mount.c:353:lustre_start_mgc()) kmalloced '(data)': 192 at ffff988d08276780. 00000020:00000010:1.0:1773941497.289223:0:14577:0:(obd_mount.c:162:do_lcfg()) kmalloced '(lcfg)': 88 at ffff988d209389c0. 00000100:00000010:1.0:1773941497.289230:0:14577:0:(lustre_peer.c:71:class_add_uuid()) kmalloced '(data)': 704 at ffff988d1081d000. 00000400:00000200:1.0:1773941497.289236:0:14577:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:1.0:1773941497.289241:0:14577:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.204.114@tcp 00000400:00000200:1.0:1773941497.289265:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941497.289272:0:14577:0:(peer.c:1737:lnet_peer_add()) peer 192.168.204.114@tcp NID flags 0x100001: -17 00000400:00000200:1.0:1773941497.289279:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941497.289288:0:14577:0:(peer.c:2767:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:1.0:1773941497.289292:0:14577:0:(peer.c:2776:lnet_discover_peer_locked()) peer 192.168.204.114@tcp NID 192.168.204.114@tcp: 0. pending discovery 00000400:00000200:1.0:1773941497.289296:0:14577:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.204.114@tcp 00000100:00000010:1.0:1773941497.289299:0:14577:0:(lustre_peer.c:113:class_add_uuid()) kfreed 'data': 704 at ffff988d1081d000. 00000020:00000010:1.0:1773941497.289304:0:14577:0:(obd_mount.c:168:do_lcfg()) kfreed 'lcfg': 88 at ffff988d209389c0. 00000020:00000010:1.0:1773941497.289311:0:14577:0:(obd_mount.c:487:lustre_start_mgc()) kmalloced '(uuid)': 40 at ffff988d209c1f80. 00000020:00000010:1.0:1773941497.289319:0:14577:0:(obd_mount.c:162:do_lcfg()) kmalloced '(lcfg)': 120 at ffff988d0716bc00. 00000020:00000010:1.0:1773941497.289343:0:14577:0:(genops.c:51:obd_device_alloc()) slab-alloced 'obd': 6584 at ffff988d083919b8. 00000020:00000010:1.0:1773941497.289352:0:14577:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff988d08f0d800. 00000020:00000010:1.0:1773941497.290665:0:14577:0:(obd_mount.c:168:do_lcfg()) kfreed 'lcfg': 120 at ffff988d0716bc00. 00000020:00000010:1.0:1773941497.290673:0:14577:0:(obd_mount.c:162:do_lcfg()) kmalloced '(lcfg)': 104 at ffff988d0716bc00. 00000001:00000010:1.0:1773941497.290695:0:14577:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at ffff988d02cfd100. 00000001:00000010:1.0:1773941497.290701:0:14577:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at ffff988d0300eca0. 00000001:00000010:1.0:1773941497.290705:0:14577:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff988d065c4a00. 00000001:00000010:1.0:1773941497.290708:0:14577:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff988d065c4600. 00000001:00000010:1.0:1773941497.290711:0:14577:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff988d065c5a00. 00000001:00000010:1.0:1773941497.290715:0:14577:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff988d065c4800. 00000020:00000010:1.0:1773941497.290719:0:14577:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d065c5000. 00000020:00000010:1.0:1773941497.290724:0:14577:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at ffff988d0716ba00. 00000020:00000010:1.0:1773941497.290730:0:14577:0:(tgt_main.c:756:tgt_ses_key_init()) slab-alloced 'session': 200 at ffff988d1827eaf0. 00008000:00000010:1.0:1773941497.290745:0:14577:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff988d1891f770. 00000020:00000010:1.0:1773941497.290752:0:14577:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d065c5800. 00000020:00000010:1.0:1773941497.290755:0:14577:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d065c5400. 00000040:00000010:1.0:1773941497.290759:0:14577:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d065c4200. 00000020:00000010:1.0:1773941497.290764:0:14577:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d08f0e800. 00000020:00000010:1.0:1773941497.290768:0:14577:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d065c4c00. 00040000:00000010:1.0:1773941497.290773:0:14577:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d02cfdf00. 00040000:00000010:1.0:1773941497.290777:0:14577:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d065c4e00. 10000000:00000010:1.0:1773941497.290781:0:14577:0:(mgc_request.c:852:mgc_device_alloc()) kmalloced '(lu)': 64 at ffff988d209c17c0. 00010000:00000010:1.0:1773941497.290794:0:14577:0:(ldlm_lib.c:421:client_obd_setup()) kmalloced '(cli->cl_read_io_latency_by_size)': 3920 at ffff988d035d6000. 00010000:00000010:1.0:1773941497.290801:0:14577:0:(ldlm_lib.c:434:client_obd_setup()) kmalloced '(cli->cl_write_io_latency_by_size)': 3920 at ffff988d035d1000. 00010000:00000010:1.0:1773941497.290809:0:14577:0:(ldlm_lockd.c:3356:ldlm_setup()) kmalloced '(ldlm_state)': 32 at ffff988d0300e900. 00000100:00000010:1.0:1773941497.290889:0:14577:0:(service.c:745:ptlrpc_register_service()) kmalloced '(service)': 360 at ffff988d07ebae00. 00000100:00000010:1.0:1773941497.290896:0:14577:0:(service.c:803:ptlrpc_register_service()) kmalloced '(svcpt)': 784 at ffff988d1081e000. 00000100:00000010:1.0:1773941497.290913:0:14577:0:(service.c:627:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_array)': 12080 at ffff988d18b40000. 00000100:00000010:1.0:1773941497.290923:0:14577:0:(service.c:635:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_count)': 3020 at ffff988d035d2000. 00000100:00000010:1.0:1773941497.290929:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1081c800. 00000100:00000010:1.0:1773941497.290934:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d18a18000. 00000100:00000010:1.0:1773941497.290938:0:14577:0:(service.c:803:ptlrpc_register_service()) kmalloced '(svcpt)': 784 at ffff988d1081d400. 00000100:00000010:1.0:1773941497.290952:0:14577:0:(service.c:627:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_array)': 12080 at ffff988d203a8000. 00000100:00000010:1.0:1773941497.290977:0:14577:0:(service.c:635:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_count)': 3020 at ffff988d035d0000. 00000100:00000010:1.0:1773941497.290984:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1081e400. 00000100:00000010:1.0:1773941497.290989:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d18a1c000. 00000400:00000200:1.0:1773941497.290995:0:14577:0:(lib-ptl.c:921:LNetSetLazyPortal()) Setting portal 15 lazy 00000020:00000010:1.0:1773941497.291035:0:14577:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff988d1081dc00. 00000020:00000010:1.0:1773941497.291040:0:14577:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 3680 at ffff988d035d3000. 00000100:00000010:1.0:1773941497.291084:0:14577:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d08276f00. 00000100:00000010:1.0:1773941497.291090:0:14577:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at ffff988d209c1fc0. 00000100:00000010:1.0:1773941497.291095:0:14577:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d08276c00. 00000100:00000010:1.0:1773941497.291099:0:14577:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d082763c0. 00000100:00000010:1.0:1773941497.291102:0:14577:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d08276900. 00000100:00000010:1.0:1773941497.291113:0:14577:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d08276000. 00000100:00000010:1.0:1773941497.291116:0:14577:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at ffff988d209c1a40. 00000100:00000010:1.0:1773941497.291119:0:14577:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d08276a80. 00000100:00000010:1.0:1773941497.291123:0:14577:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d0705b780. 00000100:00000010:1.0:1773941497.291126:0:14577:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d1a040840. 00000100:00000200:1.0:1773941497.291160:0:14577:0:(service.c:848:ptlrpc_register_service()) ldlm_cbd: Started, listening on portal 15 00000100:00000010:1.0:1773941497.291165:0:14577:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff988d07ebbc00. 00000100:00000010:1.0:1773941497.291433:0:14578:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff988d209c10c0. 00000020:00000010:1.0:1773941497.291439:0:14578:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff988d209c1600. 00000020:00000010:1.0:1773941497.291446:0:14578:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d0a135a00. 00000020:00000010:1.0:1773941497.291449:0:14578:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d20904c00. 00000040:00000010:1.0:1773941497.291453:0:14578:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d20905c00. 00000020:00000010:1.0:1773941497.291456:0:14578:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d08f0d000. 00000020:00000010:1.0:1773941497.291461:0:14578:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff988d181f69a8. 00000020:00000010:1.0:1773941497.291464:0:14578:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d20905400. 80000000:00000010:1.0:1773941497.291467:0:14578:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d20938420. 40000000:00000010:1.0:1773941497.291471:0:14578:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff988d209c1080. 00100000:00000010:1.0:1773941497.291476:0:14578:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff988d08f0c000. 00040000:00000010:1.0:1773941497.291480:0:14578:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d02cfdd00. 00040000:00000010:1.0:1773941497.291484:0:14578:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff988d1a075800. 00040000:00000010:1.0:1773941497.291487:0:14578:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d20904a00. 00000004:00000010:1.0:1773941497.291492:0:14578:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff988d1977c000. 00000004:00000010:1.0:1773941497.291497:0:14578:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff988d035d7000. 00000004:00000010:1.0:1773941497.291501:0:14578:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff988d1977b000. 00002000:00000010:1.0:1773941497.291505:0:14578:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff988d1081c400. 00000004:00000010:1.0:1773941497.291510:0:14578:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff988d1081cc00. 00000100:00000200:1.0:1773941497.291515:0:14578:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_cbd: registering portal 15 00000400:00000010:1.0:1773941497.291520:0:14578:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d87790. 00000400:00000010:1.0:1773941497.291525:0:14578:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d035dac00. 00000100:00000010:1.0:1773941497.291533:0:14578:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff988d1977c800. 00000100:00000200:1.0:1773941497.291550:0:14578:0:(service.c:3086:ptlrpc_main()) service thread 0 (#1) started 00000100:00000010:3.0:1773941497.291572:0:14577:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff988d05493400. 00000100:00000010:1.0:1773941497.291753:0:14579:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff988d209c13c0. 00000020:00000010:1.0:1773941497.291757:0:14579:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff988d209c1200. 00000020:00000010:1.0:1773941497.291762:0:14579:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d1a238400. 00000020:00000010:1.0:1773941497.291772:0:14579:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d1a239e00. 00000040:00000010:1.0:1773941497.291777:0:14579:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d19734c00. 00000020:00000010:1.0:1773941497.291781:0:14579:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d1977b800. 00000020:00000010:1.0:1773941497.291785:0:14579:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff988d181f0f18. 00000020:00000010:1.0:1773941497.291788:0:14579:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d19734800. 80000000:00000010:1.0:1773941497.291791:0:14579:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d209383c0. 40000000:00000010:1.0:1773941497.291793:0:14579:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff988d209c1880. 00100000:00000010:1.0:1773941497.291796:0:14579:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff988d1977e000. 00040000:00000010:1.0:1773941497.291799:0:14579:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d02cfda00. 00040000:00000010:1.0:1773941497.291802:0:14579:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff988d19779800. 00040000:00000010:1.0:1773941497.291805:0:14579:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d19735200. 00000004:00000010:1.0:1773941497.291808:0:14579:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff988d1977f000. 00000004:00000010:1.0:1773941497.291813:0:14579:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff988d0834d000. 00000004:00000010:1.0:1773941497.291816:0:14579:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff988d1977f800. 00002000:00000010:1.0:1773941497.291819:0:14579:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff988d1081e800. 00000004:00000010:1.0:1773941497.291822:0:14579:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff988d18159c00. 00000100:00000010:1.0:1773941497.291826:0:14579:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff988d1977a800. 00000100:00000200:1.0:1773941497.291841:0:14579:0:(service.c:3086:ptlrpc_main()) service thread 1 (#2) started 00000100:00000010:3.0:1773941497.291859:0:14577:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff988d05492000. 00000100:00000010:2.0:1773941497.292073:0:14580:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff988d05437c80. 00000020:00000010:2.0:1773941497.292078:0:14580:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff988d054370c0. 00000020:00000010:2.0:1773941497.292081:0:14580:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d10400400. 00000020:00000010:2.0:1773941497.292083:0:14580:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d10400a00. 00000040:00000010:2.0:1773941497.292086:0:14580:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d10401400. 00000020:00000010:2.0:1773941497.292088:0:14580:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d09303800. 00000020:00000010:2.0:1773941497.292091:0:14580:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff988d05400a10. 00000020:00000010:2.0:1773941497.292093:0:14580:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d10400200. 80000000:00000010:2.0:1773941497.292095:0:14580:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d08bbb960. 40000000:00000010:2.0:1773941497.292097:0:14580:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff988d05437dc0. 00100000:00000010:2.0:1773941497.292100:0:14580:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff988d09307000. 00040000:00000010:2.0:1773941497.292102:0:14580:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d0a0dbd00. 00040000:00000010:2.0:1773941497.292104:0:14580:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff988d09306000. 00040000:00000010:2.0:1773941497.292106:0:14580:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d10401e00. 00000004:00000010:2.0:1773941497.292109:0:14580:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff988d09305000. 00000004:00000010:2.0:1773941497.292111:0:14580:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff988d08afb000. 00000004:00000010:2.0:1773941497.292114:0:14580:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff988d09301000. 00002000:00000010:2.0:1773941497.292116:0:14580:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff988d035a4000. 00000004:00000010:2.0:1773941497.292118:0:14580:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff988d035a4800. 00000100:00000200:2.0:1773941497.292120:0:14580:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_cbd: registering portal 15 00000400:00000010:2.0:1773941497.292128:0:14580:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203796e0. 00000400:00000010:2.0:1773941497.292131:0:14580:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d21c4c240. 00000100:00000010:2.0:1773941497.292135:0:14580:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff988d09304800. 00000100:00000200:2.0:1773941497.292146:0:14580:0:(service.c:3086:ptlrpc_main()) service thread 0 (#1) started 00000100:00000010:3.0:1773941497.292155:0:14577:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff988d05493c00. 00000100:00000010:3.0:1773941497.292323:0:14581:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff988d02f4dac0. 00000020:00000010:3.0:1773941497.292327:0:14581:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff988d02f4d8c0. 00000020:00000010:3.0:1773941497.292332:0:14581:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d05493000. 00000020:00000010:3.0:1773941497.292336:0:14581:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d063a8000. 00000040:00000010:3.0:1773941497.292340:0:14581:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d063a9c00. 00000020:00000010:3.0:1773941497.292344:0:14581:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d0835f000. 00000020:00000010:3.0:1773941497.292348:0:14581:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff988d033469a8. 00000020:00000010:3.0:1773941497.292352:0:14581:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d063a9000. 80000000:00000010:3.0:1773941497.292355:0:14581:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d18350660. 40000000:00000010:3.0:1773941497.292357:0:14581:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff988d02f4d880. 00100000:00000010:3.0:1773941497.292360:0:14581:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff988d0835c000. 00040000:00000010:3.0:1773941497.292364:0:14581:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d02cba100. 00040000:00000010:3.0:1773941497.292368:0:14581:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff988d08359000. 00040000:00000010:3.0:1773941497.292371:0:14581:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d063a9400. 00000004:00000010:3.0:1773941497.292374:0:14581:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff988d0835a000. 00000004:00000010:3.0:1773941497.292379:0:14581:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff988d02d6c000. 00000004:00000010:3.0:1773941497.292383:0:14581:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff988d0835b800. 00002000:00000010:3.0:1773941497.292386:0:14581:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff988d054e3400. 00000004:00000010:3.0:1773941497.292389:0:14581:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff988d054e2400. 00000100:00000010:3.0:1773941497.292393:0:14581:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff988d0835d000. 00000100:00000200:3.0:1773941497.292406:0:14581:0:(service.c:3086:ptlrpc_main()) service thread 1 (#2) started 00000100:00000010:1.0:1773941497.292420:0:14577:0:(service.c:745:ptlrpc_register_service()) kmalloced '(service)': 360 at ffff988d19735000. 00000100:00000010:1.0:1773941497.292426:0:14577:0:(service.c:803:ptlrpc_register_service()) kmalloced '(svcpt)': 784 at ffff988d18159400. 00000100:00000010:1.0:1773941497.292439:0:14577:0:(service.c:627:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_array)': 12080 at ffff988d08950000. 00000100:00000010:1.0:1773941497.292447:0:14577:0:(service.c:635:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_count)': 3020 at ffff988d08349000. 00000100:00000010:1.0:1773941497.292452:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1815a000. 00000100:00000010:1.0:1773941497.292457:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d18a1a000. 00000100:00000010:1.0:1773941497.292460:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d18158800. 00000100:00000010:1.0:1773941497.292464:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d18a1e000. 00000100:00000010:1.0:1773941497.292468:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1815bc00. 00000100:00000010:1.0:1773941497.292473:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d07fd0000. 00000100:00000010:1.0:1773941497.292476:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d18159000. 00000100:00000010:1.0:1773941497.292480:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d07fd4000. 00000100:00000010:1.0:1773941497.292483:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d18159800. 00000100:00000010:1.0:1773941497.292487:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d07fd6000. 00000100:00000010:1.0:1773941497.292495:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1815ac00. 00000100:00000010:1.0:1773941497.292499:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d07fd2000. 00000100:00000010:1.0:1773941497.292502:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d18158400. 00000100:00000010:1.0:1773941497.292507:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08a00000. 00000100:00000010:1.0:1773941497.292511:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1815a400. 00000100:00000010:1.0:1773941497.292515:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08a04000. 00000100:00000010:1.0:1773941497.292518:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1815b400. 00000100:00000010:1.0:1773941497.292522:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08a06000. 00000100:00000010:1.0:1773941497.292525:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d18158000. 00000100:00000010:1.0:1773941497.292530:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08a02000. 00000100:00000010:1.0:1773941497.292532:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1815a800. 00000100:00000010:1.0:1773941497.292537:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d111fc000. 00000100:00000010:1.0:1773941497.292540:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d18158c00. 00000100:00000010:1.0:1773941497.292544:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d111fe000. 00000100:00000010:1.0:1773941497.292548:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1815b800. 00000100:00000010:1.0:1773941497.292553:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d111fa000. 00000100:00000010:1.0:1773941497.292556:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1815b000. 00000100:00000010:1.0:1773941497.292560:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d111f8000. 00000100:00000010:1.0:1773941497.292564:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d107d0000. 00000100:00000010:1.0:1773941497.292569:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08a1e000. 00000100:00000010:1.0:1773941497.292573:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d107d2800. 00000100:00000010:1.0:1773941497.292577:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08a1a000. 00000100:00000010:1.0:1773941497.292580:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d107d0c00. 00000100:00000010:1.0:1773941497.292584:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08a18000. 00000100:00000010:1.0:1773941497.292587:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d107d3800. 00000100:00000010:1.0:1773941497.292592:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08a1c000. 00000100:00000010:1.0:1773941497.292595:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d107d3000. 00000100:00000010:1.0:1773941497.292600:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08a10000. 00000100:00000010:1.0:1773941497.292603:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d107d1c00. 00000100:00000010:1.0:1773941497.292608:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08a14000. 00000100:00000010:1.0:1773941497.292611:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d107d1400. 00000100:00000010:1.0:1773941497.292620:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08a16000. 00000100:00000010:1.0:1773941497.292624:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d107d2000. 00000100:00000010:1.0:1773941497.292628:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08a12000. 00000100:00000010:1.0:1773941497.292632:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d107d0800. 00000100:00000010:1.0:1773941497.292637:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08a08000. 00000100:00000010:1.0:1773941497.292640:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d107d3c00. 00000100:00000010:1.0:1773941497.292644:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08a0c000. 00000100:00000010:1.0:1773941497.292647:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d107d1000. 00000100:00000010:1.0:1773941497.292652:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08a0e000. 00000100:00000010:1.0:1773941497.292654:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d107d1800. 00000100:00000010:1.0:1773941497.292658:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08a0a000. 00000100:00000010:1.0:1773941497.292661:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d107d2c00. 00000100:00000010:1.0:1773941497.292665:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d03320000. 00000100:00000010:1.0:1773941497.292668:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d107d0400. 00000100:00000010:1.0:1773941497.292672:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d03324000. 00000100:00000010:1.0:1773941497.292675:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d107d2400. 00000100:00000010:1.0:1773941497.292678:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d03326000. 00000100:00000010:1.0:1773941497.292681:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d107d3400. 00000100:00000010:1.0:1773941497.292685:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d03322000. 00000100:00000010:1.0:1773941497.292689:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d19724400. 00000100:00000010:1.0:1773941497.292694:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d06224000. 00000100:00000010:1.0:1773941497.292697:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d19727800. 00000100:00000010:1.0:1773941497.292702:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d06226000. 00000100:00000010:1.0:1773941497.292706:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1125a400. 00000100:00000010:1.0:1773941497.292710:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d06222000. 00000100:00000010:1.0:1773941497.292713:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1125ac00. 00000100:00000010:1.0:1773941497.292725:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d089e2000. 00000100:00000010:1.0:1773941497.292728:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1125a800. 00000100:00000010:1.0:1773941497.292732:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d089e6000. 00000100:00000010:1.0:1773941497.292734:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d11258000. 00000100:00000010:1.0:1773941497.292738:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d089e4000. 00000100:00000010:1.0:1773941497.292746:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1125b000. 00000100:00000010:1.0:1773941497.292750:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d089e0000. 00000100:00000010:1.0:1773941497.292752:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1125b800. 00000100:00000010:1.0:1773941497.292762:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08350000. 00000100:00000010:1.0:1773941497.292765:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d11259400. 00000100:00000010:1.0:1773941497.292769:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08352000. 00000100:00000010:1.0:1773941497.292772:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d11258c00. 00000100:00000010:1.0:1773941497.292776:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08356000. 00000100:00000010:1.0:1773941497.292780:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d02da7c00. 00000100:00000010:1.0:1773941497.292784:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08354000. 00000100:00000010:1.0:1773941497.292787:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d0923f400. 00000100:00000010:1.0:1773941497.292798:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d111be000. 00000100:00000010:1.0:1773941497.292801:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d0923d400. 00000100:00000010:1.0:1773941497.292805:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d111bc000. 00000100:00000010:1.0:1773941497.292808:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d0923d000. 00000100:00000010:1.0:1773941497.292812:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d111b8000. 00000100:00000010:1.0:1773941497.292814:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d0923e400. 00000100:00000010:1.0:1773941497.292818:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d111ba000. 00000100:00000010:1.0:1773941497.292821:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d0923e800. 00000100:00000010:1.0:1773941497.292830:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d089fa000. 00000100:00000010:1.0:1773941497.292833:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d0923ec00. 00000100:00000010:1.0:1773941497.292837:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d089fe000. 00000100:00000010:1.0:1773941497.292840:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d0923cc00. 00000100:00000010:1.0:1773941497.292844:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d089fc000. 00000100:00000010:1.0:1773941497.292846:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d0923f800. 00000100:00000010:1.0:1773941497.292851:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d089f8000. 00000100:00000010:1.0:1773941497.292853:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d0923c800. 00000100:00000010:1.0:1773941497.292863:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d18a32000. 00000100:00000010:1.0:1773941497.292866:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d0923d800. 00000100:00000010:1.0:1773941497.292871:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d18a36000. 00000100:00000010:1.0:1773941497.292874:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d0923c400. 00000100:00000010:1.0:1773941497.292883:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d18a34000. 00000100:00000010:1.0:1773941497.292886:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d0923fc00. 00000100:00000010:1.0:1773941497.292890:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d18a30000. 00000100:00000010:1.0:1773941497.292893:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d0923c000. 00000100:00000010:1.0:1773941497.292903:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d10814000. 00000100:00000010:1.0:1773941497.292907:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d0923dc00. 00000100:00000010:1.0:1773941497.292910:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d10810000. 00000100:00000010:1.0:1773941497.292914:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d03074800. 00000100:00000010:1.0:1773941497.292918:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d10812000. 00000100:00000010:1.0:1773941497.292923:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d11273800. 00000100:00000010:1.0:1773941497.292928:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d10816000. 00000100:00000010:1.0:1773941497.292931:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d181d0400. 00000100:00000010:1.0:1773941497.292941:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d03506000. 00000100:00000010:1.0:1773941497.292944:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d181d1800. 00000100:00000010:1.0:1773941497.292949:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d03504000. 00000100:00000010:1.0:1773941497.292951:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d181d1400. 00000100:00000010:1.0:1773941497.292956:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d03500000. 00000100:00000010:1.0:1773941497.292959:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d181d0000. 00000100:00000010:1.0:1773941497.293002:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d03502000. 00000100:00000010:1.0:1773941497.293006:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d181d2c00. 00000100:00000010:1.0:1773941497.293017:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d0357e000. 00000100:00000010:1.0:1773941497.293020:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d181d3400. 00000100:00000010:1.0:1773941497.293024:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d0357c000. 00000100:00000010:1.0:1773941497.293026:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d181d2400. 00000100:00000010:1.0:1773941497.293029:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d03578000. 00000100:00000010:1.0:1773941497.293031:0:14577:0:(service.c:803:ptlrpc_register_service()) kmalloced '(svcpt)': 784 at ffff988d181d0c00. 00000100:00000010:1.0:1773941497.293041:0:14577:0:(service.c:627:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_array)': 12080 at ffff988d089c0000. 00000100:00000010:1.0:1773941497.293049:0:14577:0:(service.c:635:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_count)': 3020 at ffff988d0834e000. 00000100:00000010:1.0:1773941497.293053:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d181d2800. 00000100:00000010:1.0:1773941497.293058:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d0357a000. 00000100:00000010:1.0:1773941497.293061:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d181d3c00. 00000100:00000010:1.0:1773941497.293079:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d07138000. 00000100:00000010:1.0:1773941497.293083:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d181d1000. 00000100:00000010:1.0:1773941497.293087:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d0713a000. 00000100:00000010:1.0:1773941497.293090:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d181d2000. 00000100:00000010:1.0:1773941497.293094:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d0713e000. 00000100:00000010:1.0:1773941497.293097:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d181d0800. 00000100:00000010:1.0:1773941497.293102:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d0713c000. 00000100:00000010:1.0:1773941497.293105:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d181d3800. 00000100:00000010:1.0:1773941497.293116:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08a2c000. 00000100:00000010:1.0:1773941497.293120:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d181d3000. 00000100:00000010:1.0:1773941497.293125:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08a28000. 00000100:00000010:1.0:1773941497.293130:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1a052000. 00000100:00000010:1.0:1773941497.293135:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08a2a000. 00000100:00000010:1.0:1773941497.293138:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1a92e000. 00000100:00000010:1.0:1773941497.293143:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08a2e000. 00000100:00000010:1.0:1773941497.293146:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1a92c800. 00000100:00000010:1.0:1773941497.293157:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d18a0e000. 00000100:00000010:1.0:1773941497.293160:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1a92fc00. 00000100:00000010:1.0:1773941497.293164:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d18a0c000. 00000100:00000010:1.0:1773941497.293168:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1a3b7c00. 00000100:00000010:1.0:1773941497.293172:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d18a08000. 00000100:00000010:1.0:1773941497.293176:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d21caf000. 00000100:00000010:1.0:1773941497.293181:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d18a0a000. 00000100:00000010:1.0:1773941497.293184:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d21caec00. 00000100:00000010:1.0:1773941497.293192:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08aec000. 00000100:00000010:1.0:1773941497.293194:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d21caf400. 00000100:00000010:1.0:1773941497.293197:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08ae8000. 00000100:00000010:1.0:1773941497.293200:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d42041c00. 00000100:00000010:1.0:1773941497.293203:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08aea000. 00000100:00000010:1.0:1773941497.293205:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d42041400. 00000100:00000010:1.0:1773941497.293208:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08aee000. 00000100:00000010:1.0:1773941497.293213:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d42043c00. 00000100:00000010:1.0:1773941497.293221:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d05420000. 00000100:00000010:1.0:1773941497.293223:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d42041000. 00000100:00000010:1.0:1773941497.293226:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d05422000. 00000100:00000010:1.0:1773941497.293229:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d42040400. 00000100:00000010:1.0:1773941497.293233:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d05426000. 00000100:00000010:1.0:1773941497.293236:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d42042400. 00000100:00000010:1.0:1773941497.293240:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d05424000. 00000100:00000010:1.0:1773941497.293255:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d180be000. 00000100:00000010:1.0:1773941497.293267:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d065ba000. 00000100:00000010:1.0:1773941497.293271:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d180bd400. 00000100:00000010:1.0:1773941497.293276:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d065be000. 00000100:00000010:1.0:1773941497.293280:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1a1b1400. 00000100:00000010:1.0:1773941497.293284:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d065bc000. 00000100:00000010:1.0:1773941497.293287:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1a1b1000. 00000100:00000010:1.0:1773941497.293291:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d065b8000. 00000100:00000010:1.0:1773941497.293294:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1a1b1c00. 00000100:00000010:1.0:1773941497.293305:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d20324000. 00000100:00000010:1.0:1773941497.293309:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1a1b0800. 00000100:00000010:1.0:1773941497.293314:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d20320000. 00000100:00000010:1.0:1773941497.293317:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1a1b3c00. 00000100:00000010:1.0:1773941497.293322:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d20322000. 00000100:00000010:1.0:1773941497.293326:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d02ce7400. 00000100:00000010:1.0:1773941497.293330:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d20326000. 00000100:00000010:1.0:1773941497.293333:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d02ce7000. 00000100:00000010:1.0:1773941497.293344:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d06518000. 00000100:00000010:1.0:1773941497.293348:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d02ce6000. 00000100:00000010:1.0:1773941497.293353:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d0651a000. 00000100:00000010:1.0:1773941497.293356:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d02ce5800. 00000100:00000010:1.0:1773941497.293361:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d0651e000. 00000100:00000010:1.0:1773941497.293364:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d02ce6800. 00000100:00000010:1.0:1773941497.293374:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d0651c000. 00000100:00000010:1.0:1773941497.293377:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d02ce4800. 00000100:00000010:1.0:1773941497.293388:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d065a6000. 00000100:00000010:1.0:1773941497.293393:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d02ce5000. 00000100:00000010:1.0:1773941497.293397:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d065a4000. 00000100:00000010:1.0:1773941497.293400:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d02ce7c00. 00000100:00000010:1.0:1773941497.293404:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d065a0000. 00000100:00000010:1.0:1773941497.293407:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d02ce4c00. 00000100:00000010:1.0:1773941497.293410:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d065a2000. 00000100:00000010:1.0:1773941497.293413:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d02ce6400. 00000100:00000010:1.0:1773941497.293425:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d18904000. 00000100:00000010:1.0:1773941497.293429:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d02ce5c00. 00000100:00000010:1.0:1773941497.293434:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d18900000. 00000100:00000010:1.0:1773941497.293437:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d02ce7800. 00000100:00000010:1.0:1773941497.293441:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d18902000. 00000100:00000010:1.0:1773941497.293445:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d02ce6c00. 00000100:00000010:1.0:1773941497.293450:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d18906000. 00000100:00000010:1.0:1773941497.293453:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d02ce5400. 00000100:00000010:1.0:1773941497.293465:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d0650c000. 00000100:00000010:1.0:1773941497.293469:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d02ce4400. 00000100:00000010:1.0:1773941497.293474:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d06508000. 00000100:00000010:1.0:1773941497.293479:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d203e7c00. 00000100:00000010:1.0:1773941497.293484:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d0650a000. 00000100:00000010:1.0:1773941497.293487:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d203e5000. 00000100:00000010:1.0:1773941497.293492:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d0650e000. 00000100:00000010:1.0:1773941497.293495:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d203e6400. 00000100:00000010:1.0:1773941497.293506:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d0897e000. 00000100:00000010:1.0:1773941497.293510:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d203e7400. 00000100:00000010:1.0:1773941497.293514:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d0897c000. 00000100:00000010:1.0:1773941497.293518:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d203e6c00. 00000100:00000010:1.0:1773941497.293522:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08978000. 00000100:00000010:1.0:1773941497.293531:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d203e6000. 00000100:00000010:1.0:1773941497.293536:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d0897a000. 00000100:00000010:1.0:1773941497.293539:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d203e4800. 00000100:00000010:1.0:1773941497.293550:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d089d4000. 00000100:00000010:1.0:1773941497.293554:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d203e7000. 00000100:00000010:1.0:1773941497.293558:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d089d0000. 00000100:00000010:1.0:1773941497.293561:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d203e4c00. 00000100:00000010:1.0:1773941497.293566:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d089d2000. 00000100:00000010:1.0:1773941497.293570:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d203e5800. 00000100:00000010:1.0:1773941497.293574:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d089d6000. 00000100:00000010:1.0:1773941497.293586:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d035cec00. 00000100:00000010:1.0:1773941497.293597:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d18914000. 00000100:00000010:1.0:1773941497.293601:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d035cd800. 00000100:00000010:1.0:1773941497.293606:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d18910000. 00000100:00000010:1.0:1773941497.293609:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d035cd000. 00000100:00000010:1.0:1773941497.293613:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d18912000. 00000100:00000010:1.0:1773941497.293616:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d035cfc00. 00000100:00000010:1.0:1773941497.293620:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d18916000. 00000100:00000010:1.0:1773941497.293623:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d035cc800. 00000100:00000010:1.0:1773941497.293634:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d02d16000. 00000100:00000010:1.0:1773941497.293638:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d035ce000. 00000100:00000010:1.0:1773941497.293642:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d02d14000. 00000100:00000010:1.0:1773941497.293645:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d035cd400. 00000100:00000010:1.0:1773941497.293650:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d02d10000. 00000100:00000010:1.0:1773941497.293653:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d035cdc00. 00000100:00000010:1.0:1773941497.293657:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d02d12000. 00000100:00000010:1.0:1773941497.293661:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d035cf000. 00000100:00000010:1.0:1773941497.293672:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d0352e000. 00000100:00000010:1.0:1773941497.293676:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d035cf800. 00000100:00000010:1.0:1773941497.293681:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d0352c000. 00000100:00000010:1.0:1773941497.293683:0:14577:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d035ccc00. 00000100:00000010:1.0:1773941497.293693:0:14577:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d03528000. 00000400:00000200:1.0:1773941497.293697:0:14577:0:(lib-ptl.c:921:LNetSetLazyPortal()) Setting portal 17 lazy 00000020:00000010:1.0:1773941497.293737:0:14577:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff988d035ce800. 00000020:00000010:1.0:1773941497.293741:0:14577:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 3680 at ffff988d0834f000. 00000100:00000010:1.0:1773941497.293790:0:14577:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d035da300. 00000100:00000010:1.0:1773941497.293794:0:14577:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at ffff988d209c1a80. 00000100:00000010:1.0:1773941497.293798:0:14577:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d035dacc0. 00000100:00000010:1.0:1773941497.293802:0:14577:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d035daa80. 00000100:00000010:1.0:1773941497.293805:0:14577:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d035dad80. 00000100:00000010:1.0:1773941497.293809:0:14577:0:(nrs.c:1026:nrs_svcpt_setup_locked()) kmalloced '(svcpt->scp_nrs_hp)': 112 at ffff988d0716b700. 00000100:00000010:1.0:1773941497.293813:0:14577:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d2030b6c0. 00000100:00000010:1.0:1773941497.293815:0:14577:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at ffff988d209c1cc0. 00000100:00000010:1.0:1773941497.293819:0:14577:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d2030bcc0. 00000100:00000010:1.0:1773941497.293821:0:14577:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d2030b900. 00000100:00000010:1.0:1773941497.293824:0:14577:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d2030be40. 00000100:00000010:1.0:1773941497.293827:0:14577:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d2030b0c0. 00000100:00000010:1.0:1773941497.293830:0:14577:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at ffff988d209c1b00. 00000100:00000010:1.0:1773941497.293833:0:14577:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d2030b180. 00000100:00000010:1.0:1773941497.293836:0:14577:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d2030b540. 00000100:00000010:1.0:1773941497.293838:0:14577:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d2030b9c0. 00000100:00000010:1.0:1773941497.293841:0:14577:0:(nrs.c:1026:nrs_svcpt_setup_locked()) kmalloced '(svcpt->scp_nrs_hp)': 112 at ffff988d0716be00. 00000100:00000010:1.0:1773941497.293844:0:14577:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d2030b600. 00000100:00000010:1.0:1773941497.293846:0:14577:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at ffff988d209c1140. 00000100:00000010:1.0:1773941497.293849:0:14577:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d2030bf00. 00000100:00000010:1.0:1773941497.293852:0:14577:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d2030b840. 00000100:00000010:1.0:1773941497.293855:0:14577:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d2030b3c0. 00000100:00000200:1.0:1773941497.293890:0:14577:0:(service.c:848:ptlrpc_register_service()) ldlm_canceld: Started, listening on portal 17 00000100:00000010:1.0:1773941497.293895:0:14577:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff988d1a247800. 00000100:00000010:1.0:1773941497.294082:0:14582:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff988d03e7cb80. 00000020:00000010:1.0:1773941497.294087:0:14582:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff988d03e7c0c0. 00000020:00000010:1.0:1773941497.294092:0:14582:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d1a246a00. 00000020:00000010:1.0:1773941497.294095:0:14582:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d1a246c00. 00000020:00000010:1.0:1773941497.294099:0:14582:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d20938d80. 00000040:00000010:1.0:1773941497.294109:0:14582:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d1a246200. 00000020:00000010:1.0:1773941497.294113:0:14582:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d19779000. 00000020:00000010:1.0:1773941497.294117:0:14582:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff988d181f2840. 00000020:00000010:1.0:1773941497.294121:0:14582:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d02df8200. 80000000:00000010:1.0:1773941497.294124:0:14582:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d20938a20. 40000000:00000010:1.0:1773941497.294126:0:14582:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff988d03e7c4c0. 00008000:00000010:1.0:1773941497.294131:0:14582:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d0833a350. 00100000:00000010:1.0:1773941497.294135:0:14582:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff988d21759000. 00040000:00000010:1.0:1773941497.294139:0:14582:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d02cfd900. 00040000:00000010:1.0:1773941497.294142:0:14582:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff988d1a9ad800. 00040000:00000010:1.0:1773941497.294146:0:14582:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d0a32f800. 00000004:00000010:1.0:1773941497.294150:0:14582:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff988d1a9ae800. 00000004:00000010:1.0:1773941497.294154:0:14582:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff988d0834b000. 00000004:00000010:1.0:1773941497.294158:0:14582:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff988d19763800. 00002000:00000010:1.0:1773941497.294161:0:14582:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff988d035cc000. 00000004:00000010:1.0:1773941497.294165:0:14582:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff988d035cf400. 00000100:00000200:1.0:1773941497.294168:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294173:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d87420. 00000400:00000010:1.0:1773941497.294177:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d2030b780. 00000100:00000200:1.0:1773941497.294180:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294183:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d87bb0. 00000400:00000010:1.0:1773941497.294185:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d2030b480. 00000100:00000200:1.0:1773941497.294188:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294191:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d87210. 00000400:00000010:1.0:1773941497.294193:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d2030bc00. 00000100:00000200:1.0:1773941497.294196:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294198:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d87528. 00000400:00000010:1.0:1773941497.294201:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d2030b300. 00000100:00000200:1.0:1773941497.294203:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294205:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d87268. 00000400:00000010:1.0:1773941497.294208:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d2030ba80. 00000100:00000200:1.0:1773941497.294210:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294212:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d87058. 00000400:00000010:1.0:1773941497.294215:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d2030b000. 00000100:00000200:1.0:1773941497.294225:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294228:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d87ec8. 00000400:00000010:1.0:1773941497.294231:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d2030b240. 00000100:00000200:1.0:1773941497.294233:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294235:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d872c0. 00000400:00000010:1.0:1773941497.294238:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d2030bb40. 00000100:00000200:1.0:1773941497.294240:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294258:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d87948. 00000400:00000010:1.0:1773941497.294262:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d02dcbb40. 00000100:00000200:1.0:1773941497.294265:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294268:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d879a0. 00000400:00000010:1.0:1773941497.294271:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d21dd1f00. 00000100:00000200:1.0:1773941497.294273:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294276:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d874d0. 00000400:00000010:1.0:1773941497.294279:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d034cc240. 00000100:00000200:1.0:1773941497.294281:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294284:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d87c08. 00000400:00000010:1.0:1773941497.294287:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d2081a480. 00000100:00000200:1.0:1773941497.294289:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294292:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d87630. 00000400:00000010:1.0:1773941497.294294:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d2081a0c0. 00000100:00000200:1.0:1773941497.294296:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294299:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d879f8. 00000400:00000010:1.0:1773941497.294301:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d2081ad80. 00000100:00000200:1.0:1773941497.294304:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294306:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d87318. 00000400:00000010:1.0:1773941497.294310:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d06510c00. 00000100:00000200:1.0:1773941497.294313:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294316:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d87e70. 00000400:00000010:1.0:1773941497.294318:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d06510540. 00000100:00000200:1.0:1773941497.294320:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294323:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d87898. 00000400:00000010:1.0:1773941497.294325:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d06510000. 00000100:00000200:1.0:1773941497.294327:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294330:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d87b00. 00000400:00000010:1.0:1773941497.294340:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d06510240. 00000100:00000200:1.0:1773941497.294344:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294346:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d87e18. 00000400:00000010:1.0:1773941497.294349:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d06510900. 00000100:00000200:1.0:1773941497.294351:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294354:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d871b8. 00000400:00000010:1.0:1773941497.294356:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d06510180. 00000100:00000200:1.0:1773941497.294358:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294361:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d87c60. 00000400:00000010:1.0:1773941497.294363:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d06510300. 00000100:00000200:1.0:1773941497.294365:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294367:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d87000. 00000400:00000010:1.0:1773941497.294370:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d06510480. 00000100:00000200:1.0:1773941497.294372:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294374:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d87cb8. 00000400:00000010:1.0:1773941497.294377:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d06510e40. 00000100:00000200:1.0:1773941497.294380:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294382:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d876e0. 00000400:00000010:1.0:1773941497.294385:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d06510d80. 00000100:00000200:1.0:1773941497.294387:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294390:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d87b58. 00000400:00000010:1.0:1773941497.294392:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d06510cc0. 00000100:00000200:1.0:1773941497.294395:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294397:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d87f20. 00000400:00000010:1.0:1773941497.294400:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d065100c0. 00000100:00000200:1.0:1773941497.294402:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294405:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d87d68. 00000400:00000010:1.0:1773941497.294407:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d065106c0. 00000100:00000200:1.0:1773941497.294410:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294412:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d878f0. 00000400:00000010:1.0:1773941497.294415:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d06510f00. 00000100:00000200:1.0:1773941497.294417:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294419:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d873c8. 00000400:00000010:1.0:1773941497.294422:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d065103c0. 00000100:00000200:1.0:1773941497.294424:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294434:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d87dc0. 00000400:00000010:1.0:1773941497.294437:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d06510a80. 00000100:00000200:1.0:1773941497.294440:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294442:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d870b0. 00000400:00000010:1.0:1773941497.294445:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d06510b40. 00000100:00000200:1.0:1773941497.294447:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294449:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d87738. 00000400:00000010:1.0:1773941497.294452:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d06510780. 00000100:00000200:1.0:1773941497.294454:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294456:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d87478. 00000400:00000010:1.0:1773941497.294461:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d04c81840. 00000100:00000200:1.0:1773941497.294463:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294465:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d87688. 00000400:00000010:1.0:1773941497.294468:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d04c810c0. 00000100:00000200:1.0:1773941497.294470:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294472:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d87f78. 00000400:00000010:1.0:1773941497.294475:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a03cf00. 00000100:00000200:1.0:1773941497.294477:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294480:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d877e8. 00000400:00000010:1.0:1773941497.294483:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a03c180. 00000100:00000200:1.0:1773941497.294485:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294487:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d87370. 00000400:00000010:1.0:1773941497.294489:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a03ccc0. 00000100:00000200:1.0:1773941497.294492:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294494:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d87840. 00000400:00000010:1.0:1773941497.294497:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a03c600. 00000100:00000200:1.0:1773941497.294499:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294501:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d21d875d8. 00000400:00000010:1.0:1773941497.294504:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a03c240. 00000100:00000200:1.0:1773941497.294506:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294509:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c318. 00000400:00000010:1.0:1773941497.294512:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a03c840. 00000100:00000200:1.0:1773941497.294514:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294517:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1ce70. 00000400:00000010:1.0:1773941497.294520:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d20863300. 00000100:00000200:1.0:1773941497.294529:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294533:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c898. 00000400:00000010:1.0:1773941497.294535:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d20863180. 00000100:00000200:1.0:1773941497.294538:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294540:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1cb00. 00000400:00000010:1.0:1773941497.294542:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d20863840. 00000100:00000200:1.0:1773941497.294544:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294547:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1ce18. 00000400:00000010:1.0:1773941497.294549:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d20863e40. 00000100:00000200:1.0:1773941497.294552:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294554:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c1b8. 00000400:00000010:1.0:1773941497.294557:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d20863a80. 00000100:00000200:1.0:1773941497.294559:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294562:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1cc60. 00000400:00000010:1.0:1773941497.294564:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d20863540. 00000100:00000200:1.0:1773941497.294566:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294569:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c000. 00000400:00000010:1.0:1773941497.294571:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d20863900. 00000100:00000200:1.0:1773941497.294573:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294575:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1ccb8. 00000400:00000010:1.0:1773941497.294578:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d20863480. 00000100:00000200:1.0:1773941497.294580:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294582:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c6e0. 00000400:00000010:1.0:1773941497.294585:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d208639c0. 00000100:00000200:1.0:1773941497.294587:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294589:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1cb58. 00000400:00000010:1.0:1773941497.294591:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d20863f00. 00000100:00000200:1.0:1773941497.294594:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294596:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1cf20. 00000400:00000010:1.0:1773941497.294598:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d20863780. 00000100:00000200:1.0:1773941497.294600:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294603:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1cd68. 00000400:00000010:1.0:1773941497.294608:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a0333c0. 00000100:00000200:1.0:1773941497.294610:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294613:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c8f0. 00000400:00000010:1.0:1773941497.294622:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a033cc0. 00000100:00000200:1.0:1773941497.294625:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294628:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c3c8. 00000400:00000010:1.0:1773941497.294631:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d08e34600. 00000100:00000200:1.0:1773941497.294633:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294635:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1cdc0. 00000400:00000010:1.0:1773941497.294638:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d08e34300. 00000100:00000200:1.0:1773941497.294640:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294643:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c0b0. 00000400:00000010:1.0:1773941497.294645:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d08e34e40. 00000100:00000200:1.0:1773941497.294647:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294650:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c738. 00000400:00000010:1.0:1773941497.294652:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d08e34900. 00000100:00000200:1.0:1773941497.294655:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294657:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c478. 00000400:00000010:1.0:1773941497.294660:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d08e34f00. 00000100:00000200:1.0:1773941497.294662:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294665:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c688. 00000400:00000010:1.0:1773941497.294667:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d08e34cc0. 00000100:00000200:1.0:1773941497.294669:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294671:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1cf78. 00000400:00000010:1.0:1773941497.294673:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d08e34840. 00000100:00000200:1.0:1773941497.294676:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294678:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1ca50. 00000400:00000010:1.0:1773941497.294681:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d08e343c0. 00000100:00000200:1.0:1773941497.294683:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294685:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c108. 00000400:00000010:1.0:1773941497.294689:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d063e4d80. 00000100:00000200:1.0:1773941497.294691:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294693:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1caa8. 00000400:00000010:1.0:1773941497.294696:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d063e46c0. 00000100:00000200:1.0:1773941497.294698:0:14582:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:1.0:1773941497.294700:0:14582:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c160. 00000400:00000010:1.0:1773941497.294703:0:14582:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d063e4540. 00000100:00000010:1.0:1773941497.294707:0:14582:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff988d19767800. 00000100:00000200:1.0:1773941497.294726:0:14582:0:(service.c:3086:ptlrpc_main()) service thread 0 (#1) started 00000100:00000010:3.0:1773941497.294763:0:14577:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff988d0642c200. 00000100:00000010:1.0:1773941497.294945:0:14583:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff988d03e7cdc0. 00000020:00000010:1.0:1773941497.294949:0:14583:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff988d03e7c480. 00000020:00000010:1.0:1773941497.294955:0:14583:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d0a130000. 00000020:00000010:1.0:1773941497.295159:0:14583:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d181fc400. 00000020:00000010:1.0:1773941497.295164:0:14583:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d2024ade0. 00000040:00000010:1.0:1773941497.295168:0:14583:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d181fd800. 00000020:00000010:1.0:1773941497.295182:0:14583:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d08996000. 00000020:00000010:1.0:1773941497.295187:0:14583:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff988d181f5a90. 00000020:00000010:1.0:1773941497.295190:0:14583:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d181fd600. 80000000:00000010:1.0:1773941497.295193:0:14583:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d2024a600. 40000000:00000010:1.0:1773941497.295196:0:14583:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff988d03e7c240. 00008000:00000010:1.0:1773941497.295199:0:14583:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d083391a8. 00100000:00000010:1.0:1773941497.295202:0:14583:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff988d08997800. 00040000:00000010:1.0:1773941497.295205:0:14583:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d02cfd800. 00040000:00000010:1.0:1773941497.295208:0:14583:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff988d08997000. 00040000:00000010:1.0:1773941497.295211:0:14583:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d181fc800. 00000004:00000010:1.0:1773941497.295214:0:14583:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff988d08994000. 00000004:00000010:1.0:1773941497.295218:0:14583:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff988d02bb9000. 00000004:00000010:1.0:1773941497.295222:0:14583:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff988d08993000. 00002000:00000010:1.0:1773941497.295225:0:14583:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff988d035ce400. 00000004:00000010:1.0:1773941497.295228:0:14583:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff988d035cc400. 00000100:00000010:1.0:1773941497.295232:0:14583:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff988d08994800. 00000100:00000200:1.0:1773941497.295262:0:14583:0:(service.c:3086:ptlrpc_main()) service thread 1 (#2) started 00000100:00000010:3.0:1773941497.295277:0:14577:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff988d0642d800. 00000100:00000010:1.0:1773941497.295435:0:14584:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff988d03e7c800. 00000020:00000010:1.0:1773941497.295440:0:14584:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff988d03e7cc40. 00000020:00000010:1.0:1773941497.295444:0:14584:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d181fc000. 00000020:00000010:1.0:1773941497.295448:0:14584:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d181fce00. 00000020:00000010:1.0:1773941497.295463:0:14584:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d054f3300. 00000040:00000010:1.0:1773941497.295468:0:14584:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d181fd000. 00000020:00000010:1.0:1773941497.295471:0:14584:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d08995800. 00000020:00000010:1.0:1773941497.295475:0:14584:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff988d181f0000. 00000020:00000010:1.0:1773941497.295478:0:14584:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d181fc200. 80000000:00000010:1.0:1773941497.295480:0:14584:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d054f3000. 40000000:00000010:1.0:1773941497.295488:0:14584:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff988d03e7c840. 00008000:00000010:1.0:1773941497.295492:0:14584:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d0833b880. 00100000:00000010:1.0:1773941497.295495:0:14584:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff988d08996800. 00040000:00000010:1.0:1773941497.295498:0:14584:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d02cfd200. 00040000:00000010:1.0:1773941497.295501:0:14584:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff988d08995000. 00040000:00000010:1.0:1773941497.295504:0:14584:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d181fc600. 00000004:00000010:1.0:1773941497.295507:0:14584:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff988d08990000. 00000004:00000010:1.0:1773941497.295512:0:14584:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff988d0396b000. 00000004:00000010:1.0:1773941497.295515:0:14584:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff988d08990800. 00002000:00000010:1.0:1773941497.295527:0:14584:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff988d1118b800. 00000004:00000010:1.0:1773941497.295530:0:14584:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff988d11188c00. 00000100:00000010:1.0:1773941497.295535:0:14584:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff988d08992000. 00000100:00000200:1.0:1773941497.295547:0:14584:0:(service.c:3086:ptlrpc_main()) service thread 2 (#3) started 00000100:00000010:3.0:1773941497.295563:0:14577:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff988d0642d200. 00000100:00000010:2.0:1773941497.295730:0:14585:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff988d05437fc0. 00000020:00000010:2.0:1773941497.295733:0:14585:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff988d054378c0. 00000020:00000010:2.0:1773941497.295736:0:14585:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d02d18400. 00000020:00000010:2.0:1773941497.295738:0:14585:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d02d18a00. 00000020:00000010:2.0:1773941497.295740:0:14585:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d08bbb600. 00000040:00000010:2.0:1773941497.295742:0:14585:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d02d18600. 00000020:00000010:2.0:1773941497.295746:0:14585:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d181c1800. 00000020:00000010:2.0:1773941497.295748:0:14585:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff988d05400508. 00000020:00000010:2.0:1773941497.295750:0:14585:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d02d18000. 80000000:00000010:2.0:1773941497.295752:0:14585:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d08bbb1e0. 40000000:00000010:2.0:1773941497.295754:0:14585:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff988d05437240. 00008000:00000010:2.0:1773941497.295757:0:14585:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d083266d8. 00100000:00000010:2.0:1773941497.295759:0:14585:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff988d181c1000. 00040000:00000010:2.0:1773941497.295761:0:14585:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d0a0db200. 00040000:00000010:2.0:1773941497.295764:0:14585:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff988d181c2000. 00040000:00000010:2.0:1773941497.295766:0:14585:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d02d18c00. 00000004:00000010:2.0:1773941497.295768:0:14585:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff988d181c0800. 00000004:00000010:2.0:1773941497.295770:0:14585:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff988d08aff000. 00000004:00000010:2.0:1773941497.295773:0:14585:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff988d181c0000. 00002000:00000010:2.0:1773941497.295775:0:14585:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff988d035a7400. 00000004:00000010:2.0:1773941497.295777:0:14585:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff988d035a5c00. 00000100:00000200:2.0:1773941497.295779:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295789:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379000. 00000400:00000010:2.0:1773941497.295792:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d21c4c300. 00000100:00000200:2.0:1773941497.295795:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295796:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379cb8. 00000400:00000010:2.0:1773941497.295801:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d21c4c6c0. 00000100:00000200:2.0:1773941497.295803:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295804:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379b58. 00000400:00000010:2.0:1773941497.295806:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d21c4c000. 00000100:00000200:2.0:1773941497.295812:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295813:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379f20. 00000400:00000010:2.0:1773941497.295815:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d21c4c780. 00000100:00000200:2.0:1773941497.295816:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295818:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379d68. 00000400:00000010:2.0:1773941497.295819:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d21c4c3c0. 00000100:00000200:2.0:1773941497.295820:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295822:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203798f0. 00000400:00000010:2.0:1773941497.295823:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d21c4c840. 00000100:00000200:2.0:1773941497.295824:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295826:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203793c8. 00000400:00000010:2.0:1773941497.295827:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d21c4ca80. 00000100:00000200:2.0:1773941497.295828:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295829:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379dc0. 00000400:00000010:2.0:1773941497.295830:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d21c4cf00. 00000100:00000200:2.0:1773941497.295832:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295833:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203790b0. 00000400:00000010:2.0:1773941497.295834:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d21c4c180. 00000100:00000200:2.0:1773941497.295835:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295836:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379738. 00000400:00000010:2.0:1773941497.295837:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d21c4c900. 00000100:00000200:2.0:1773941497.295839:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295840:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379478. 00000400:00000010:2.0:1773941497.295841:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d21c4cc00. 00000100:00000200:2.0:1773941497.295842:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295843:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379688. 00000400:00000010:2.0:1773941497.295846:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18b98cc0. 00000100:00000200:2.0:1773941497.295847:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295848:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379f78. 00000400:00000010:2.0:1773941497.295850:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18b98b40. 00000100:00000200:2.0:1773941497.295851:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295852:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379a50. 00000400:00000010:2.0:1773941497.295853:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18b98e40. 00000100:00000200:2.0:1773941497.295855:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295856:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379108. 00000400:00000010:2.0:1773941497.295860:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18b98240. 00000100:00000200:2.0:1773941497.295862:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295863:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379aa8. 00000400:00000010:2.0:1773941497.295865:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18b98300. 00000100:00000200:2.0:1773941497.295866:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295868:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379160. 00000400:00000010:2.0:1773941497.295869:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18b98f00. 00000100:00000200:2.0:1773941497.295870:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295871:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379d10. 00000400:00000010:2.0:1773941497.295873:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18b98600. 00000100:00000200:2.0:1773941497.295874:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295875:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379580. 00000400:00000010:2.0:1773941497.295876:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18b98000. 00000100:00000200:2.0:1773941497.295877:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295878:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000400:00000010:2.0:1773941497.295880:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18b98180. 00000100:00000200:2.0:1773941497.295881:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295882:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203797e8. 00000400:00000010:2.0:1773941497.295883:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18b98d80. 00000100:00000200:2.0:1773941497.295884:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295885:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379840. 00000400:00000010:2.0:1773941497.295887:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18b980c0. 00000100:00000200:2.0:1773941497.295888:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295889:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379370. 00000400:00000010:2.0:1773941497.295890:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18b98a80. 00000100:00000200:2.0:1773941497.295891:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295892:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379790. 00000400:00000010:2.0:1773941497.295894:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18b989c0. 00000100:00000200:2.0:1773941497.295895:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295896:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379420. 00000400:00000010:2.0:1773941497.295897:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18b986c0. 00000100:00000200:2.0:1773941497.295898:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295899:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379bb0. 00000400:00000010:2.0:1773941497.295900:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18b98780. 00000100:00000200:2.0:1773941497.295901:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295906:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379210. 00000400:00000010:2.0:1773941497.295907:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18b98900. 00000100:00000200:2.0:1773941497.295909:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295910:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379528. 00000400:00000010:2.0:1773941497.295912:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18b98c00. 00000100:00000200:2.0:1773941497.295913:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295914:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379268. 00000400:00000010:2.0:1773941497.295915:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18b98480. 00000100:00000200:2.0:1773941497.295916:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295917:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379058. 00000400:00000010:2.0:1773941497.295919:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18b98540. 00000100:00000200:2.0:1773941497.295920:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295921:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379ec8. 00000400:00000010:2.0:1773941497.295922:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18b983c0. 00000100:00000200:2.0:1773941497.295923:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295924:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203792c0. 00000400:00000010:2.0:1773941497.295926:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18b98840. 00000100:00000200:2.0:1773941497.295927:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295928:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379948. 00000400:00000010:2.0:1773941497.295930:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a9dd600. 00000100:00000200:2.0:1773941497.295931:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295932:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203799a0. 00000400:00000010:2.0:1773941497.295934:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a9dd900. 00000100:00000200:2.0:1773941497.295935:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295936:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203794d0. 00000400:00000010:2.0:1773941497.295937:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a9ddd80. 00000100:00000200:2.0:1773941497.295938:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295940:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379c08. 00000400:00000010:2.0:1773941497.295941:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a9dd9c0. 00000100:00000200:2.0:1773941497.295942:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295943:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379630. 00000400:00000010:2.0:1773941497.295944:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a9dd300. 00000100:00000200:2.0:1773941497.295945:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295947:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203799f8. 00000400:00000010:2.0:1773941497.295948:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a9ddc00. 00000100:00000200:2.0:1773941497.295952:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295954:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379318. 00000400:00000010:2.0:1773941497.295955:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a9dde40. 00000100:00000200:2.0:1773941497.295956:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295957:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379e70. 00000400:00000010:2.0:1773941497.295958:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a9ddf00. 00000100:00000200:2.0:1773941497.295979:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295982:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379898. 00000400:00000010:2.0:1773941497.295984:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a9dd6c0. 00000100:00000200:2.0:1773941497.295986:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295988:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379b00. 00000400:00000010:2.0:1773941497.295989:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a9dd840. 00000100:00000200:2.0:1773941497.295991:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.295994:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379e18. 00000400:00000010:2.0:1773941497.295996:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a9dd180. 00000100:00000200:2.0:1773941497.295998:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.296000:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379c60. 00000400:00000010:2.0:1773941497.296003:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a9dd540. 00000100:00000200:2.0:1773941497.296004:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.296005:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203791b8. 00000400:00000010:2.0:1773941497.296007:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a9dda80. 00000100:00000200:2.0:1773941497.296009:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.296019:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e478. 00000400:00000010:2.0:1773941497.296021:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a9dd3c0. 00000100:00000200:2.0:1773941497.296023:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.296024:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e688. 00000400:00000010:2.0:1773941497.296026:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a9dd0c0. 00000100:00000200:2.0:1773941497.296027:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.296028:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380ef78. 00000400:00000010:2.0:1773941497.296029:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a9dd780. 00000100:00000200:2.0:1773941497.296030:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.296031:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380ea50. 00000400:00000010:2.0:1773941497.296033:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a9ddcc0. 00000100:00000200:2.0:1773941497.296034:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.296035:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e108. 00000400:00000010:2.0:1773941497.296041:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a9dd240. 00000100:00000200:2.0:1773941497.296043:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.296044:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380eaa8. 00000400:00000010:2.0:1773941497.296046:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a9ddb40. 00000100:00000200:2.0:1773941497.296047:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.296048:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e160. 00000400:00000010:2.0:1773941497.296050:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18807d80. 00000100:00000200:2.0:1773941497.296051:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.296052:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380ed10. 00000400:00000010:2.0:1773941497.296054:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18807c00. 00000100:00000200:2.0:1773941497.296055:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.296056:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e580. 00000400:00000010:2.0:1773941497.296057:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18807a80. 00000100:00000200:2.0:1773941497.296058:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.296060:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e5d8. 00000400:00000010:2.0:1773941497.296061:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d188073c0. 00000100:00000200:2.0:1773941497.296062:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.296063:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e7e8. 00000400:00000010:2.0:1773941497.296065:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18807180. 00000100:00000200:2.0:1773941497.296066:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.296067:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e840. 00000400:00000010:2.0:1773941497.296068:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d188079c0. 00000100:00000200:2.0:1773941497.296069:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.296070:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e370. 00000400:00000010:2.0:1773941497.296072:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18807480. 00000100:00000200:2.0:1773941497.296073:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.296074:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e790. 00000400:00000010:2.0:1773941497.296075:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18807600. 00000100:00000200:2.0:1773941497.296076:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.296078:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e420. 00000400:00000010:2.0:1773941497.296079:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18807e40. 00000100:00000200:2.0:1773941497.296080:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.296081:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380ebb0. 00000400:00000010:2.0:1773941497.296082:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18807240. 00000100:00000200:2.0:1773941497.296083:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.296087:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e210. 00000400:00000010:2.0:1773941497.296089:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18807000. 00000100:00000200:2.0:1773941497.296090:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.296092:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e528. 00000400:00000010:2.0:1773941497.296093:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18807780. 00000100:00000200:2.0:1773941497.296094:0:14585:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941497.296095:0:14585:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e268. 00000400:00000010:2.0:1773941497.296097:0:14585:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18807cc0. 00000100:00000010:2.0:1773941497.296099:0:14585:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff988d181c3000. 00000100:00000200:2.0:1773941497.296109:0:14585:0:(service.c:3086:ptlrpc_main()) service thread 0 (#1) started 00000100:00000010:3.0:1773941497.296136:0:14577:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff988d04f8ae00. 00000100:00000010:3.0:1773941497.296328:0:14586:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff988d02f4dc80. 00000020:00000010:3.0:1773941497.296333:0:14586:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff988d02f4dd00. 00000020:00000010:3.0:1773941497.296339:0:14586:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d0335ce00. 00000020:00000010:3.0:1773941497.296342:0:14586:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d0335c400. 00000020:00000010:3.0:1773941497.296345:0:14586:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d18350600. 00000040:00000010:3.0:1773941497.296348:0:14586:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d0335d400. 00000020:00000010:3.0:1773941497.296358:0:14586:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d08358000. 00000020:00000010:3.0:1773941497.296362:0:14586:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff988d03341e30. 00000020:00000010:3.0:1773941497.296365:0:14586:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d0335d800. 80000000:00000010:3.0:1773941497.296368:0:14586:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d18350360. 40000000:00000010:3.0:1773941497.296371:0:14586:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff988d02f4d000. 00008000:00000010:3.0:1773941497.296375:0:14586:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d107f3c08. 00100000:00000010:3.0:1773941497.296378:0:14586:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff988d08358800. 00040000:00000010:3.0:1773941497.296382:0:14586:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d02cba000. 00040000:00000010:3.0:1773941497.296385:0:14586:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff988d08359800. 00040000:00000010:3.0:1773941497.296388:0:14586:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d0335c800. 00000004:00000010:3.0:1773941497.296392:0:14586:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff988d0835e800. 00000004:00000010:3.0:1773941497.296396:0:14586:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff988d02d6f000. 00000004:00000010:3.0:1773941497.296400:0:14586:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff988d0835d800. 00002000:00000010:3.0:1773941497.296403:0:14586:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff988d054e1c00. 00000004:00000010:3.0:1773941497.296407:0:14586:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff988d054e2800. 00000100:00000010:3.0:1773941497.296411:0:14586:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff988d0835a800. 00000100:00000200:3.0:1773941497.296427:0:14586:0:(service.c:3086:ptlrpc_main()) service thread 1 (#2) started 00000100:00000010:1.0:1773941497.296450:0:14577:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff988d181fd400. 00000100:00000010:2.0:1773941497.296627:0:14587:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff988d05437180. 00000020:00000010:2.0:1773941497.296631:0:14587:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff988d05437740. 00000020:00000010:2.0:1773941497.296635:0:14587:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d02d18e00. 00000020:00000010:2.0:1773941497.296638:0:14587:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d02d19a00. 00000020:00000010:2.0:1773941497.296641:0:14587:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d08bbbf60. 00000040:00000010:2.0:1773941497.296644:0:14587:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d1a1c8c00. 00000020:00000010:2.0:1773941497.296647:0:14587:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d181c4000. 00000020:00000010:2.0:1773941497.296649:0:14587:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff988d05403758. 00000020:00000010:2.0:1773941497.296651:0:14587:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d1a1c8200. 80000000:00000010:2.0:1773941497.296653:0:14587:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d08bbba80. 40000000:00000010:2.0:1773941497.296655:0:14587:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff988d05437a00. 00008000:00000010:2.0:1773941497.296657:0:14587:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d08325fc8. 00100000:00000010:2.0:1773941497.296659:0:14587:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff988d181c7000. 00040000:00000010:2.0:1773941497.296661:0:14587:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d0a0dbb00. 00040000:00000010:2.0:1773941497.296664:0:14587:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff988d181c7800. 00040000:00000010:2.0:1773941497.296665:0:14587:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d1a1c8600. 00000004:00000010:2.0:1773941497.296668:0:14587:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff988d181c5800. 00000004:00000010:2.0:1773941497.296670:0:14587:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff988d08afc000. 00000004:00000010:2.0:1773941497.296673:0:14587:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff988d181c4800. 00002000:00000010:2.0:1773941497.296675:0:14587:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff988d035a7c00. 00000004:00000010:2.0:1773941497.296676:0:14587:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff988d035a4c00. 00000100:00000010:2.0:1773941497.296679:0:14587:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff988d181c5000. 00000100:00000200:2.0:1773941497.296688:0:14587:0:(service.c:3086:ptlrpc_main()) service thread 2 (#3) started 00010000:00000010:3.0:1773941497.296704:0:14577:0:(ldlm_lockd.c:3466:ldlm_setup()) kmalloced '(blp)': 168 at ffff988d02eb6780. 00010000:00000010:1.0:1773941497.296840:0:14588:0:(ldlm_lockd.c:2961:ldlm_bl_thread_main()) kmalloced '(env)': 56 at ffff988d03e7c940. 00000020:00000010:1.0:1773941497.296846:0:14588:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d1961e200. 00000020:00000010:1.0:1773941497.296850:0:14588:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d1961ea00. 00000020:00000010:1.0:1773941497.296853:0:14588:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d08991000. 00000020:00000010:1.0:1773941497.296857:0:14588:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff988d181f4168. 00000020:00000010:1.0:1773941497.296860:0:14588:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d1961fa00. 80000000:00000010:1.0:1773941497.296863:0:14588:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d054f3ea0. 40000000:00000010:1.0:1773941497.296866:0:14588:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff988d03e7cd40. 00100000:00000010:1.0:1773941497.296870:0:14588:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff988d08991800. 00040000:00000010:1.0:1773941497.296873:0:14588:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d02cfdc00. 00040000:00000010:1.0:1773941497.296886:0:14588:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d0335e800. 00002000:00000010:1.0:1773941497.296891:0:14588:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff988d1118a800. 00000020:00000010:1.0:1773941497.296894:0:14588:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff988d03e7c740. 00010000:00000010:1.0:1773941497.297057:0:14589:0:(ldlm_lockd.c:2961:ldlm_bl_thread_main()) kmalloced '(env)': 56 at ffff988d03e7c640. 00000020:00000010:1.0:1773941497.297060:0:14589:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d0335ea00. 00000020:00000010:1.0:1773941497.297062:0:14589:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d0335fc00. 00000020:00000010:1.0:1773941497.297065:0:14589:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d08992800. 00000020:00000010:1.0:1773941497.297067:0:14589:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff988d181f3758. 00000020:00000010:1.0:1773941497.297069:0:14589:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d0335fe00. 80000000:00000010:1.0:1773941497.297076:0:14589:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d054f3120. 40000000:00000010:1.0:1773941497.297077:0:14589:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff988d03e7c3c0. 00100000:00000010:1.0:1773941497.297080:0:14589:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff988d08993800. 00040000:00000010:1.0:1773941497.297082:0:14589:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d02cfd700. 00040000:00000010:1.0:1773941497.297084:0:14589:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d0335fa00. 00002000:00000010:1.0:1773941497.297086:0:14589:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff988d11188000. 00000020:00000010:1.0:1773941497.297088:0:14589:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff988d03e7cf80. 00000020:00000010:1.0:1773941497.297213:0:14590:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d0335e000. 00000020:00000010:1.0:1773941497.297215:0:14590:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d0335f200. 00000040:00000010:1.0:1773941497.297217:0:14590:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d0335f600. 00000020:00000010:1.0:1773941497.297228:0:14590:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d18b1c000. 00000020:00000010:1.0:1773941497.297231:0:14590:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff988d181f3250. 00000020:00000010:1.0:1773941497.297233:0:14590:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d0335e400. 80000000:00000010:1.0:1773941497.297235:0:14590:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d054f3c00. 40000000:00000010:1.0:1773941497.297236:0:14590:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff988d03e7c380. 00100000:00000010:1.0:1773941497.297239:0:14590:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff988d18b1b000. 00040000:00000010:1.0:1773941497.297240:0:14590:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d02cfd000. 00040000:00000010:1.0:1773941497.297258:0:14590:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff988d18b1c800. 00040000:00000010:1.0:1773941497.297261:0:14590:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d0335f400. 00000004:00000010:1.0:1773941497.297263:0:14590:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff988d18b1d800. 00000004:00000010:1.0:1773941497.297268:0:14590:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff988d065f6000. 00000004:00000010:1.0:1773941497.297270:0:14590:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff988d18b1e800. 00002000:00000010:1.0:1773941497.297272:0:14590:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff988d1118b400. 00000004:00000010:1.0:1773941497.297275:0:14590:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff988d1118a400. 00000020:00000010:1.0:1773941497.297277:0:14590:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff988d03e7c580. 00000001:00000010:3.0:1773941497.297301:0:14577:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at ffff988d18350060. 00000001:00000010:3.0:1773941497.297310:0:14577:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at ffff988d183502a0. 00000020:00000010:3.0:1773941497.297331:0:14577:0:(genops.c:1283:class_new_import()) kmalloced '(imp)': 1712 at ffff988d035c4800. 00000100:00000200:3.0:1773941497.297351:0:14577:0:(events.c:560:ptlrpc_uuid_to_peer()) 192.168.204.114@tcp->12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941497.297359:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.297366:0:14577:0:(peer.c:1471:LNetPrimaryNID()) NID 192.168.204.114@tcp primary NID 192.168.204.114@tcp rc 0 00010000:00000010:3.0:1773941497.297372:0:14577:0:(ldlm_lib.c:70:import_set_conn()) kmalloced '(imp_conn)': 88 at ffff988d18350240. 00010000:00000010:3.0:1773941497.297378:0:14577:0:(ldlm_resource.c:955:ldlm_namespace_new()) kmalloced '(ns)': 848 at ffff988d054e0400. 00000001:00000010:3.0:1773941497.297382:0:14577:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 280 at ffff988d0335d200. 00000001:00000010:3.0:1773941497.297387:0:14577:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 16 at ffff988d0633c650. 00000001:00000010:3.0:1773941497.297391:0:14577:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 304 at ffff988d0335d600. 00000001:00000010:3.0:1773941497.297394:0:14577:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 304 at ffff988d0335de00. 00000020:00000010:3.0:1773941497.297454:0:14577:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff988d054e0c00. 00000020:00000010:3.0:1773941497.297458:0:14577:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 96 at ffff988d18350f00. 00000020:00000010:3.0:1773941497.297519:0:14577:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff988d054e1800. 00000020:00000010:3.0:1773941497.297528:0:14577:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 352 at ffff988d0335c000. 00000040:00000010:3.0:1773941497.297545:0:14577:0:(llog_obd.c:26:llog_new_ctxt()) kmalloced '(ctxt)': 168 at ffff988d02eb60c0. 00000020:00000010:2.0:1773941497.297835:0:14577:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d065c5400. 00000040:00000010:2.0:1773941497.297851:0:14577:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d065c4200. 00000020:00000010:2.0:1773941497.297854:0:14577:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d08f0e800. 00000020:00000010:2.0:1773941497.297860:0:14577:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d065c4c00. 00040000:00000010:2.0:1773941497.297863:0:14577:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d02cfdf00. 00040000:00000010:2.0:1773941497.297867:0:14577:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d065c4e00. 00000020:00000010:2.0:1773941497.297869:0:14577:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d065c5800. 00000020:00000010:2.0:1773941497.297873:0:14577:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at ffff988d0716ba00. 00000020:00000010:2.0:1773941497.297876:0:14577:0:(tgt_main.c:768:tgt_ses_key_fini()) slab-freed '(session)': 200 at ffff988d1827eaf0. 00008000:00000010:2.0:1773941497.297879:0:14577:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff988d1891f770. 00000020:00000010:2.0:1773941497.297882:0:14577:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d065c5000. 00000020:00000010:2.0:1773941497.297886:0:14577:0:(obd_mount.c:168:do_lcfg()) kfreed 'lcfg': 104 at ffff988d0716bc00. 00000020:00000010:2.0:1773941497.297899:0:14577:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff988d181c6800. 00000400:00000200:2.0:1773941497.297910:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941497.297915:0:14577:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000010:2.0:1773941497.297925:0:14577:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06507800. 02000000:00000010:2.0:1773941497.297939:0:14577:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d035a7000. 00000020:00000010:2.0:1773941497.297989:0:14577:0:(obd_mount.c:561:lustre_start_mgc()) kfreed 'uuid': 40 at ffff988d209c1f80. 00000020:00000010:2.0:1773941497.297993:0:14577:0:(obd_mount.c:562:lustre_start_mgc()) kfreed 'data': 192 at ffff988d08276780. 00000020:00000010:2.0:1773941497.297996:0:14577:0:(obd_mount.c:563:lustre_start_mgc()) kfreed 'mgcname': 23 at ffff988d0300eee0. 00000080:00000010:2.0:1773941497.298002:0:14577:0:(llite_lib.c:1399:ll_fill_super()) kmalloced '(cfg)': 88 at ffff988d08bbb840. 00000080:00000010:2.0:1773941497.298006:0:14577:0:(llite_lib.c:95:ll_init_sbi()) kmalloced '(sbi)': 1376 at ffff988d181c3800. 02000000:00000010:3.0:1773941497.298016:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d054e0000. 00000400:00000200:3.0:1773941497.298026:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941497.298031:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccdf20. 00000400:00000010:3.0:1773941497.298038:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ace58. 00000100:00000200:3.0:1773941497.298043:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291472384, portal 25 00000400:00000010:3.0:1773941497.298049:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acd48. 00000100:00000200:3.0:1773941497.298053:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291472384, offset 0 00000400:00000200:3.0:1773941497.298063:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000020:00000010:2.0:1773941497.298064:0:14577:0:(cl_page.c:1380:cl_cache_init()) kmalloced '(cache)': 152 at ffff988d18807840. 00000080:00000010:2.0:1773941497.298069:0:14577:0:(llite_lib.c:129:ll_init_sbi()) kmalloced '(sbi->ll_foreign_symlink_prefix)': 6 at ffff988d083a0eb8. 00000080:00000010:2.0:1773941497.298072:0:14577:0:(llite_lib.c:136:ll_init_sbi()) kmalloced '(sbi->ll_foreign_symlink_upcall)': 5 at ffff988d083a08b8. 00000400:00000200:3.0:1773941497.298073:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.298083:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941497.298091:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941497.298097:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941497.298102:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941497.298107:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941497.298113:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [78:78:256:4294967295] 192.168.204.114@tcp LPNI seq info [78:78:8:4294967295] 00000400:00000200:3.0:1773941497.298121:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.298133:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:3.0:1773941497.298141:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:3.0:1773941497.298147:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cba200. 00000800:00000200:3.0:1773941497.298156:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:3.0:1773941497.298165:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:3.0:1773941497.298171:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cba200 type 1, nob 616 niov 1 nkiov 1 00000400:00000200:3.0:1773941497.298191:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000010:3.0:1773941497.298197:0:2400:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d183504e0. 00000100:00000010:3.0:1773941497.298209:0:2400:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d18350ba0. 00000800:00000200:0.0:1773941497.298333:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941497.298340:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cba200. 00000400:00000200:0.0:1773941497.298345:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941497.298353:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941497.298359:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acd48 00000400:00000010:0.0:1773941497.298362:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acd48. 00000100:00000200:0.0:1773941497.298379:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06507800 x1860112291472384/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941507 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1773941497.298528:0:14577:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff988d035a5400. 00000020:00000010:2.0:1773941497.298533:0:14577:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 1440 at ffff988d08f0e800. 00000020:00000010:2.0:1773941497.298543:0:14577:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff988d035a6000. 00000020:00000010:2.0:1773941497.298546:0:14577:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 576 at ffff988d035a6c00. 00000020:00000010:2.0:1773941497.298554:0:14577:0:(obd_mount.c:67:lustre_process_log()) kmalloced '(bufs)': 104 at ffff988d107f4e00. 00000020:00000010:2.0:1773941497.298558:0:14577:0:(obd_mount.c:76:lustre_process_log()) kmalloced '(lcfg)': 184 at ffff988d18807300. 00000020:00000010:2.0:1773941497.298562:0:14577:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d11266a00. 00000020:00000010:2.0:1773941497.298566:0:14577:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d065c5000. 00000040:00000010:2.0:1773941497.298575:0:14577:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d065c5800. 00000020:00000010:2.0:1773941497.298586:0:14577:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d05448800. 00000020:00000010:2.0:1773941497.298592:0:14577:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d065c4e00. 00040000:00000010:2.0:1773941497.298597:0:14577:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d0a0db600. 00040000:00000010:2.0:1773941497.298602:0:14577:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d065c4c00. 10000000:00000010:2.0:1773941497.298610:0:14577:0:(mgc_request.c:197:do_config_log_add()) kmalloced '(cld)': 287 at ffff988d065c4200. 02000000:00000010:2.0:1773941497.298618:0:14577:0:(sec_config.c:546:sptlrpc_conf_get()) kmalloced '(conf)': 120 at ffff988d107f4900. 00000100:00000010:2.0:1773941497.298626:0:14577:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06504e00. 02000000:00000010:2.0:1773941497.298633:0:14577:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065c5400. 00010000:00000010:2.0:1773941497.298641:0:14577:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d071ee100. 00010000:00000010:2.0:1773941497.298648:0:14577:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d089dfa00. 00000020:00000010:2.0:1773941497.298651:0:14577:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 128 at ffff988d107f4200. 00000100:00000010:2.0:1773941497.298665:0:14577:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d188076c0. 00000020:00000010:3.0:1773941497.299034:0:1075:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 448 at ffff988d0335c200. 00000400:00000200:0.0:1773941497.299946:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.299951:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941497.299956:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.299959:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.299978:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.299983:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324602400 00000400:00000200:0.0:1773941497.299987:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.299992:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0x779 [1] + 0 00000400:00000200:0.0:1773941497.300006:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941497.300010:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:0.0:1773941497.300017:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06507800 x1860112291472384/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941507 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1773941497.300077:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccdf20. 00000400:00000200:3.0:1773941497.300090:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ace58 00000400:00000010:3.0:1773941497.300093:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ace58. 00000100:00000200:3.0:1773941497.300120:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06507800 x1860112291472384/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941507 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941497.300132:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06507800 x1860112291472384/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941507 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1773941497.300211:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d054e0000. 02000000:00000010:3.0:1773941497.300219:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d035a7000. 00000100:00000010:3.0:1773941497.300225:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06507800. 02000000:00000010:2.0:1773941497.300237:0:14577:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a5000. 00000400:00000200:2.0:1773941497.300258:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941497.300263:0:14577:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e058. 00000400:00000010:2.0:1773941497.300277:0:14577:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516aa0. 00000100:00000200:2.0:1773941497.300282:0:14577:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291472512, portal 25 00000400:00000010:2.0:1773941497.300287:0:14577:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516f68. 00000100:00000200:2.0:1773941497.300290:0:14577:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291472512, offset 0 00000400:00000200:2.0:1773941497.300297:0:14577:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941497.300303:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941497.300309:0:14577:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941497.300313:0:14577:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941497.300316:0:14577:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941497.300320:0:14577:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941497.300323:0:14577:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:2.0:1773941497.300327:0:14577:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [79:79:256:4294967295] 192.168.204.114@tcp LPNI seq info [79:79:8:4294967295] 00000400:00000200:2.0:1773941497.300333:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941497.300344:0:14577:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:2.0:1773941497.300350:0:14577:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:2.0:1773941497.300361:0:14577:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d217a8e00. 00000800:00000200:2.0:1773941497.300367:0:14577:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:2.0:1773941497.300374:0:14577:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:2.0:1773941497.300378:0:14577:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d217a8e00 type 1, nob 424 niov 1 nkiov 1 00000400:00000200:2.0:1773941497.300394:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:0.0:1773941497.300455:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941497.300460:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d217a8e00. 00000400:00000200:0.0:1773941497.300465:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941497.300471:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941497.300475:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516f68 00000400:00000010:0.0:1773941497.300477:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516f68. 00000100:00000200:0.0:1773941497.300485:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06504e00 x1860112291472512/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941513 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1773941497.301538:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.301544:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941497.301549:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.301552:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.301555:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.301559:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 344 into portal 25 MB=0x69bc324602480 00000400:00000200:0.0:1773941497.301562:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.301565:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 344/344 into md 0x781 [1] + 224 00000400:00000200:0.0:1773941497.301581:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941497.301583:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:0.0:1773941497.301590:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06504e00 x1860112291472512/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941513 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1773941497.301661:0:14577:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e058. 00000400:00000200:3.0:1773941497.301667:0:14577:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516aa0 00000400:00000010:3.0:1773941497.301669:0:14577:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516aa0. 00000100:00000200:3.0:1773941497.301681:0:14577:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06504e00 x1860112291472512/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941513 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941497.301694:0:14577:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06504e00 x1860112291472512/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941513 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1773941497.301721:0:14577:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d188076c0. 02000000:00000010:3.0:1773941497.301747:0:14577:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a5000. 02000000:00000010:3.0:1773941497.301752:0:14577:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065c5400. 00000100:00000010:3.0:1773941497.301759:0:14577:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06504e00. 10000000:00000010:3.0:1773941497.301770:0:14577:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d02f4d200. 00000020:00000010:3.0:1773941497.301775:0:14577:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d0335dc00. 00000020:00000010:3.0:1773941497.301781:0:14577:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d0335d000. 00000040:00000010:3.0:1773941497.301786:0:14577:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d0335ca00. 00000020:00000010:3.0:1773941497.301792:0:14577:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d035c2000. 00000020:00000010:3.0:1773941497.301798:0:14577:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d0335c600. 80000000:00000010:3.0:1773941497.301805:0:14577:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d18350960. 00040000:00000010:3.0:1773941497.301810:0:14577:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d0335cc00. 20000000:00000010:3.0:1773941497.301818:0:14577:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d0335da00. 00000040:00000010:3.0:1773941497.301828:0:14577:0:(llog.c:42:llog_alloc_handle()) kmalloced '(loghandle)': 352 at ffff988d065c5400. 00000100:00000010:3.0:1773941497.301836:0:14577:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d111f1500. 02000000:00000010:3.0:1773941497.301851:0:14577:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d1817d200. 00000100:00000010:3.0:1773941497.301878:0:14577:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb6a80. 02000000:00000010:3.0:1773941497.301897:0:14577:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff988d1817c000. 00000400:00000200:3.0:1773941497.301905:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941497.301910:0:14577:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccdf20. 00000400:00000010:3.0:1773941497.301917:0:14577:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ace58. 00000100:00000200:3.0:1773941497.301935:0:14577:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112291472640, portal 25 00000400:00000010:3.0:1773941497.301942:0:14577:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac220. 00000100:00000200:3.0:1773941497.301946:0:14577:0:(niobuf.c:94:ptl_send_buf()) Sending 512 bytes to portal 26, xid 1860112291472640, offset 0 00000400:00000200:3.0:1773941497.301955:0:14577:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941497.302000:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.302008:0:14577:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941497.302012:0:14577:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941497.302017:0:14577:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941497.302020:0:14577:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941497.302023:0:14577:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941497.302028:0:14577:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [80:80:256:4294967295] 192.168.204.114@tcp LPNI seq info [80:80:8:4294967295] 00000400:00000200:3.0:1773941497.302032:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.302041:0:14577:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:3.0:1773941497.302047:0:14577:0:(socklnd_cb.c:1001:ksocknal_send()) sending 512 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:3.0:1773941497.302051:0:14577:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cba300. 00000800:00000200:3.0:1773941497.302061:0:14577:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:3.0:1773941497.302067:0:14577:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:3.0:1773941497.302071:0:14577:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cba300 type 1, nob 608 niov 1 nkiov 1 00000400:00000200:3.0:1773941497.302088:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:0.0:1773941497.302131:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941497.302134:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cba300. 00000400:00000200:0.0:1773941497.302138:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941497.302142:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941497.302146:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac220 00000400:00000010:0.0:1773941497.302148:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac220. 00000100:00000200:0.0:1773941497.302154:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d111f1500 x1860112291472640/t0(0) o501->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 512/272 e 0 to 0 dl 1773941513 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1773941497.303316:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.303322:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941497.303328:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.303332:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.303337:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.303341:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 272 into portal 25 MB=0x69bc324602500 00000400:00000200:0.0:1773941497.303345:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.303350:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 272/272 into md 0x789 [1] + 224 00000400:00000200:0.0:1773941497.303362:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941497.303367:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:0.0:1773941497.303375:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d111f1500 x1860112291472640/t0(0) o501->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 512/272 e 0 to 0 dl 1773941513 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1773941497.303435:0:14577:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccdf20. 00000400:00000200:3.0:1773941497.303440:0:14577:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ace58 00000400:00000010:3.0:1773941497.303442:0:14577:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ace58. 00000100:00000200:3.0:1773941497.303450:0:14577:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d111f1500 x1860112291472640/t0(0) o501->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 512/272 e 0 to 0 dl 1773941513 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941497.303463:0:14577:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d111f1500 x1860112291472640/t0(0) o501->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 512/272 e 0 to 0 dl 1773941513 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1773941497.303500:0:14577:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb6a80. 02000000:00000010:3.0:1773941497.303510:0:14577:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff988d1817c000. 02000000:00000010:3.0:1773941497.303514:0:14577:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d1817d200. 00000100:00000010:3.0:1773941497.303517:0:14577:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d111f1500. 00000040:00000010:3.0:1773941497.303524:0:14577:0:(llog.c:72:llog_free_handle()) kfreed 'loghandle': 352 at ffff988d065c5400. 00000020:00000010:3.0:1773941497.303533:0:14577:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d0335d000. 00000040:00000010:3.0:1773941497.303540:0:14577:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d0335ca00. 00000020:00000010:3.0:1773941497.303546:0:14577:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d035c2000. 00000020:00000010:3.0:1773941497.303552:0:14577:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d0335c600. 80000000:00000010:3.0:1773941497.303555:0:14577:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d18350960. 00040000:00000010:3.0:1773941497.303559:0:14577:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d0335cc00. 20000000:00000010:3.0:1773941497.303563:0:14577:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d0335da00. 00000020:00000010:3.0:1773941497.303566:0:14577:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0335dc00. 10000000:00000010:3.0:1773941497.303570:0:14577:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d02f4d200. 10000000:00000010:3.0:1773941497.303599:0:14577:0:(mgc_request.c:197:do_config_log_add()) kmalloced '(cld)': 279 at ffff988d1817d200. 10000000:00000010:3.0:1773941497.303607:0:14577:0:(mgc_request.c:197:do_config_log_add()) kmalloced '(cld)': 286 at ffff988d1817c000. 10000000:00000010:3.0:1773941497.303614:0:14577:0:(mgc_request.c:197:do_config_log_add()) kmalloced '(cld)': 285 at ffff988d1817c800. 00000100:00000010:3.0:1773941497.303622:0:14577:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d111f1500. 02000000:00000010:3.0:1773941497.303637:0:14577:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d1817c200. 00010000:00000010:3.0:1773941497.303649:0:14577:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d0168a600. 00000020:00000010:3.0:1773941497.303654:0:14577:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 128 at ffff988d03580b00. 00000100:00000010:3.0:1773941497.303665:0:14577:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb6a80. 02000000:00000010:3.0:1773941497.303673:0:14577:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d054e0000. 00000400:00000200:3.0:1773941497.303677:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941497.303680:0:14577:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccdf20. 00000400:00000010:3.0:1773941497.303688:0:14577:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ace58. 00000100:00000200:3.0:1773941497.303691:0:14577:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291472768, portal 25 00000400:00000010:3.0:1773941497.303694:0:14577:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acaa0. 00000100:00000200:3.0:1773941497.303697:0:14577:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291472768, offset 0 00000400:00000200:3.0:1773941497.303701:0:14577:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941497.303707:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.303718:0:14577:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941497.303721:0:14577:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941497.303724:0:14577:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941497.303726:0:14577:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941497.303729:0:14577:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941497.303732:0:14577:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [81:81:256:4294967295] 192.168.204.114@tcp LPNI seq info [81:81:8:4294967295] 00000400:00000200:3.0:1773941497.303736:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.303743:0:14577:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:3.0:1773941497.303748:0:14577:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:3.0:1773941497.303752:0:14577:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cbac00. 00000800:00000200:3.0:1773941497.303758:0:14577:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:3.0:1773941497.303765:0:14577:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:3.0:1773941497.303768:0:14577:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cbac00 type 1, nob 424 niov 1 nkiov 1 00000400:00000200:3.0:1773941497.303784:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:0.0:1773941497.303827:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941497.303831:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cbac00. 00000400:00000200:0.0:1773941497.303834:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941497.303839:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941497.303842:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acaa0 00000400:00000010:0.0:1773941497.303844:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acaa0. 00000100:00000200:0.0:1773941497.303856:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d111f1500 x1860112291472768/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941513 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1773941497.304853:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.304857:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941497.304861:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.304863:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.304866:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.304868:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 344 into portal 25 MB=0x69bc324602580 00000400:00000200:0.0:1773941497.304871:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.304875:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 344/344 into md 0x791 [1] + 224 00000400:00000200:0.0:1773941497.304891:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941497.304894:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:0.0:1773941497.304899:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d111f1500 x1860112291472768/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941513 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1773941497.304953:0:14577:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccdf20. 00000400:00000200:3.0:1773941497.304996:0:14577:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ace58 00000400:00000010:3.0:1773941497.305000:0:14577:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ace58. 00000100:00000200:3.0:1773941497.305010:0:14577:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d111f1500 x1860112291472768/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941513 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941497.305019:0:14577:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d111f1500 x1860112291472768/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941513 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1773941497.305036:0:14577:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb6a80. 02000000:00000010:3.0:1773941497.305050:0:14577:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d054e0000. 02000000:00000010:3.0:1773941497.305054:0:14577:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d1817c200. 00000100:00000010:3.0:1773941497.305059:0:14577:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d111f1500. 10000000:00000010:3.0:1773941497.305067:0:14577:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d02f4d200. 00000020:00000010:3.0:1773941497.305070:0:14577:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d1817c200. 00000020:00000010:3.0:1773941497.305074:0:14577:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d1817ca00. 00000040:00000010:3.0:1773941497.305078:0:14577:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d1817ce00. 00000020:00000010:3.0:1773941497.305082:0:14577:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d035c2000. 00000020:00000010:3.0:1773941497.305093:0:14577:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d1817c600. 80000000:00000010:3.0:1773941497.305097:0:14577:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d18350960. 00040000:00000010:3.0:1773941497.305102:0:14577:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d1817cc00. 20000000:00000010:3.0:1773941497.305105:0:14577:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d1817d000. 00000040:00000010:3.0:1773941497.305111:0:14577:0:(llog.c:42:llog_alloc_handle()) kmalloced '(loghandle)': 352 at ffff988d1817d400. 00000100:00000010:3.0:1773941497.305117:0:14577:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d111f1500. 02000000:00000010:3.0:1773941497.305126:0:14577:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d1817d600. 00000100:00000010:3.0:1773941497.305135:0:14577:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb6a80. 02000000:00000010:3.0:1773941497.305146:0:14577:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff988d1817c400. 00000400:00000200:3.0:1773941497.305152:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941497.305155:0:14577:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccdf20. 00000400:00000010:3.0:1773941497.305159:0:14577:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ace58. 00000100:00000200:3.0:1773941497.305162:0:14577:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112291472896, portal 25 00000400:00000010:3.0:1773941497.305166:0:14577:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acb28. 00000100:00000200:3.0:1773941497.305168:0:14577:0:(niobuf.c:94:ptl_send_buf()) Sending 512 bytes to portal 26, xid 1860112291472896, offset 0 00000400:00000200:3.0:1773941497.305175:0:14577:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941497.305179:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.305184:0:14577:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941497.305186:0:14577:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941497.305188:0:14577:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941497.305191:0:14577:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941497.305193:0:14577:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941497.305199:0:14577:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [82:82:256:4294967295] 192.168.204.114@tcp LPNI seq info [82:82:8:4294967295] 00000400:00000200:3.0:1773941497.305202:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.305209:0:14577:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:3.0:1773941497.305214:0:14577:0:(socklnd_cb.c:1001:ksocknal_send()) sending 512 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:3.0:1773941497.305217:0:14577:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cbaf00. 00000800:00000200:3.0:1773941497.305221:0:14577:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:3.0:1773941497.305227:0:14577:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:3.0:1773941497.305230:0:14577:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cbaf00 type 1, nob 608 niov 1 nkiov 1 00000400:00000200:3.0:1773941497.305261:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:0.0:1773941497.305327:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941497.305333:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cbaf00. 00000400:00000200:0.0:1773941497.305338:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941497.305344:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941497.305347:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acb28 00000400:00000010:0.0:1773941497.305350:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acb28. 00000100:00000200:0.0:1773941497.305359:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d111f1500 x1860112291472896/t0(0) o501->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 512/272 e 0 to 0 dl 1773941513 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1773941497.306288:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.306294:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941497.306300:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.306304:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.306308:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.306312:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 272 into portal 25 MB=0x69bc324602600 00000400:00000200:0.0:1773941497.306317:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.306322:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 272/272 into md 0x799 [1] + 224 00000400:00000200:0.0:1773941497.306343:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941497.306347:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:0.0:1773941497.306355:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d111f1500 x1860112291472896/t0(0) o501->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 512/272 e 0 to 0 dl 1773941513 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1773941497.306409:0:14577:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccdf20. 00000400:00000200:3.0:1773941497.306413:0:14577:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ace58 00000400:00000010:3.0:1773941497.306415:0:14577:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ace58. 00000100:00000200:3.0:1773941497.306421:0:14577:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d111f1500 x1860112291472896/t0(0) o501->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 512/272 e 0 to 0 dl 1773941513 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941497.306429:0:14577:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d111f1500 x1860112291472896/t0(0) o501->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 512/272 e 0 to 0 dl 1773941513 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1773941497.306443:0:14577:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb6a80. 02000000:00000010:3.0:1773941497.306448:0:14577:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff988d1817c400. 02000000:00000010:3.0:1773941497.306450:0:14577:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d1817d600. 00000100:00000010:3.0:1773941497.306451:0:14577:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d111f1500. 00000040:00000010:3.0:1773941497.306460:0:14577:0:(llog.c:412:llog_init_handle()) kmalloced '(llh)': 8192 at ffff988d20306000. 00000100:00000010:3.0:1773941497.306464:0:14577:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d111f1500. 02000000:00000010:3.0:1773941497.306468:0:14577:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d1817d600. 00000100:00000010:3.0:1773941497.306473:0:14577:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb6a80. 02000000:00000010:3.0:1773941497.306494:0:14577:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 16384 at ffff988d065b0000. 00000400:00000200:3.0:1773941497.306498:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941497.306502:0:14577:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccdf20. 00000400:00000010:3.0:1773941497.306506:0:14577:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 184 at ffff988d02eb69c0. 00000100:00000200:3.0:1773941497.306510:0:14577:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 16384 bytes, xid 1860112291473024, portal 25 00000400:00000010:3.0:1773941497.306515:0:14577:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ace58. 00000100:00000200:3.0:1773941497.306519:0:14577:0:(niobuf.c:94:ptl_send_buf()) Sending 272 bytes to portal 26, xid 1860112291473024, offset 0 00000400:00000200:3.0:1773941497.306525:0:14577:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941497.306530:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.306544:0:14577:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941497.306549:0:14577:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941497.306553:0:14577:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941497.306557:0:14577:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941497.306560:0:14577:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941497.306563:0:14577:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [83:83:256:4294967295] 192.168.204.114@tcp LPNI seq info [83:83:8:4294967295] 00000400:00000200:3.0:1773941497.306567:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.306575:0:14577:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:3.0:1773941497.306586:0:14577:0:(socklnd_cb.c:1001:ksocknal_send()) sending 272 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:3.0:1773941497.306589:0:14577:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cba400. 00000800:00000200:3.0:1773941497.306593:0:14577:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:3.0:1773941497.306597:0:14577:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:3.0:1773941497.306600:0:14577:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cba400 type 1, nob 368 niov 1 nkiov 1 00000400:00000200:3.0:1773941497.306612:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:0.0:1773941497.306679:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941497.306684:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cba400. 00000400:00000200:0.0:1773941497.306688:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941497.306694:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941497.306698:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ace58 00000400:00000010:0.0:1773941497.306700:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ace58. 00000100:00000200:0.0:1773941497.306709:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d111f1500 x1860112291473024/t0(0) o503->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8416 e 0 to 0 dl 1773941513 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1773941497.307802:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.307807:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941497.307812:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.307815:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.307818:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.307821:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 8416 into portal 25 MB=0x69bc324602680 00000400:00000200:0.0:1773941497.307825:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.307828:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 8416/8416 into md 0x7a1 [4] + 224 00000400:00000200:0.0:1773941497.307842:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941497.307847:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:0.0:1773941497.307856:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d111f1500 x1860112291473024/t0(0) o503->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8416 e 0 to 0 dl 1773941513 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1773941497.307887:0:14577:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccdf20. 00000400:00000200:3.0:1773941497.307892:0:14577:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d02eb69c0 00000400:00000010:3.0:1773941497.307893:0:14577:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 184 at ffff988d02eb69c0. 00000100:00000200:3.0:1773941497.307898:0:14577:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d111f1500 x1860112291473024/t0(0) o503->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8416 e 0 to 0 dl 1773941513 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941497.307906:0:14577:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d111f1500 x1860112291473024/t0(0) o503->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8416 e 0 to 0 dl 1773941513 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1773941497.307917:0:14577:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb6a80. 02000000:00000010:3.0:1773941497.307922:0:14577:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 16384 at ffff988d065b0000. 02000000:00000010:3.0:1773941497.307931:0:14577:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d1817d600. 00000100:00000010:3.0:1773941497.307934:0:14577:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d111f1500. 00000040:00000010:3.0:1773941497.307942:0:14577:0:(llog.c:945:llog_process_or_fork()) kmalloced '(lpi)': 96 at ffff988d183505a0. 00000020:00000010:3.0:1773941497.308191:0:14596:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d1817d600. 00000020:00000010:3.0:1773941497.308196:0:14596:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d1817de00. 00000040:00000010:3.0:1773941497.308198:0:14596:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d1817dc00. 00000020:00000010:3.0:1773941497.308201:0:14596:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d035c1800. 00000020:00000010:3.0:1773941497.308206:0:14596:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff988d03340f18. 00000020:00000010:3.0:1773941497.308209:0:14596:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d1817d800. 80000000:00000010:3.0:1773941497.308211:0:14596:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d18350720. 40000000:00000010:3.0:1773941497.308215:0:14596:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff988d02f4d480. 00100000:00000010:3.0:1773941497.308219:0:14596:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff988d035c1000. 00040000:00000010:3.0:1773941497.308223:0:14596:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d02cba400. 00040000:00000010:3.0:1773941497.308229:0:14596:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff988d035c0800. 00040000:00000010:3.0:1773941497.308232:0:14596:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d1817da00. 00000004:00000010:3.0:1773941497.308236:0:14596:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff988d035c0000. 00000004:00000010:3.0:1773941497.308259:0:14596:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff988d02d6a000. 00000004:00000010:3.0:1773941497.308265:0:14596:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff988d035c5000. 00002000:00000010:3.0:1773941497.308270:0:14596:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff988d054e0000. 00000004:00000010:3.0:1773941497.308274:0:14596:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff988d054e3800. 00000020:00000010:3.0:1773941497.308278:0:14596:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff988d02f4df80. 00000040:00000010:3.0:1773941497.308287:0:14596:0:(llog.c:570:llog_process_thread()) kmalloced '(buf)': 8192 at ffff988d20302000. 00000100:00000010:3.0:1773941497.308293:0:14596:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d111f1500. 02000000:00000010:3.0:1773941497.308299:0:14596:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d1817c400. 00000100:00000010:3.0:1773941497.308305:0:14596:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb69c0. 02000000:00000010:3.0:1773941497.308319:0:14596:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 16384 at ffff988d065b0000. 00000400:00000200:3.0:1773941497.308323:0:14596:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941497.308326:0:14596:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccdf20. 00000400:00000010:3.0:1773941497.308329:0:14596:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 184 at ffff988d02eb6600. 00000100:00000200:3.0:1773941497.308331:0:14596:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 16384 bytes, xid 1860112291473152, portal 25 00000400:00000010:3.0:1773941497.308335:0:14596:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac2a8. 00000100:00000200:3.0:1773941497.308337:0:14596:0:(niobuf.c:94:ptl_send_buf()) Sending 272 bytes to portal 26, xid 1860112291473152, offset 0 00000400:00000200:3.0:1773941497.308342:0:14596:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941497.308345:0:14596:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.308350:0:14596:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941497.308352:0:14596:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941497.308354:0:14596:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941497.308356:0:14596:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941497.308358:0:14596:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941497.308361:0:14596:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [84:84:256:4294967295] 192.168.204.114@tcp LPNI seq info [84:84:8:4294967295] 00000400:00000200:3.0:1773941497.308363:0:14596:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.308368:0:14596:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:3.0:1773941497.308372:0:14596:0:(socklnd_cb.c:1001:ksocknal_send()) sending 272 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:3.0:1773941497.308375:0:14596:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cbaf00. 00000800:00000200:3.0:1773941497.308378:0:14596:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:3.0:1773941497.308382:0:14596:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:3.0:1773941497.308385:0:14596:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cbaf00 type 1, nob 368 niov 1 nkiov 1 00000400:00000200:3.0:1773941497.308402:0:14596:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:0.0:1773941497.308470:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941497.308474:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cbaf00. 00000400:00000200:0.0:1773941497.308478:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941497.308483:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941497.308486:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac2a8 00000400:00000010:0.0:1773941497.308488:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac2a8. 00000100:00000200:0.0:1773941497.308501:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d111f1500 x1860112291473152/t0(0) o502->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941513 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'llog_process_th.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1773941497.309498:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.309504:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941497.309513:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.309516:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.309519:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.309521:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 8472 into portal 25 MB=0x69bc324602700 00000400:00000200:0.0:1773941497.309525:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.309527:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 8472/8472 into md 0x7a9 [4] + 224 00000400:00000200:0.0:1773941497.309539:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941497.309542:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:0.0:1773941497.309548:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d111f1500 x1860112291473152/t0(0) o502->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941513 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'llog_process_th.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1773941497.309595:0:14596:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccdf20. 00000400:00000200:3.0:1773941497.309601:0:14596:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d02eb6600 00000400:00000010:3.0:1773941497.309604:0:14596:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 184 at ffff988d02eb6600. 00000100:00000200:3.0:1773941497.309611:0:14596:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d111f1500 x1860112291473152/t0(0) o502->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941513 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'llog_process_th.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941497.309622:0:14596:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d111f1500 x1860112291473152/t0(0) o502->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941513 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'llog_process_th.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1773941497.309637:0:14596:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb69c0. 02000000:00000010:3.0:1773941497.309642:0:14596:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 16384 at ffff988d065b0000. 02000000:00000010:3.0:1773941497.309651:0:14596:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d1817c400. 00000100:00000010:3.0:1773941497.309654:0:14596:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d111f1500. 00000020:00000010:3.0:1773941497.309669:0:14596:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at ffff988d02cbac00. 00000020:00000010:3.0:1773941497.309673:0:14596:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at ffff988d02cbac00. 00000020:00000010:3.0:1773941497.309678:0:14596:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 34 at ffff988d02f4d080. 00000020:00000010:3.0:1773941497.309681:0:14596:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 128 at ffff988d03580580. 00000020:00000010:3.0:1773941497.309695:0:14596:0:(genops.c:51:obd_device_alloc()) slab-alloced 'obd': 6584 at ffff988d089a99b8. 00000020:00000010:3.0:1773941497.309701:0:14596:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff988d035c6800. 00000020:00000010:3.0:1773941497.309716:0:14596:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 128 at ffff988d03580580. 00000020:00000010:3.0:1773941497.309718:0:14596:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 34 at ffff988d02f4d080. 00000020:00000010:3.0:1773941497.309721:0:14596:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 34 at ffff988d02f4d080. 00000020:00000010:3.0:1773941497.309723:0:14596:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 160 at ffff988d02eb69c0. 00000001:00000010:3.0:1773941497.309743:0:14596:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at ffff988d02cbac00. 00000001:00000010:3.0:1773941497.309747:0:14596:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at ffff988d107cf0e0. 00000001:00000010:3.0:1773941497.309751:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff988d0335da00. 00000001:00000010:3.0:1773941497.309754:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff988d0335cc00. 00000001:00000010:3.0:1773941497.309756:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff988d0335c600. 00000001:00000010:3.0:1773941497.309758:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff988d0335ca00. 00000020:00000010:3.0:1773941497.309760:0:14596:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d0335d000. 00000020:00000010:3.0:1773941497.309764:0:14596:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at ffff988d03580580. 00000020:00000010:3.0:1773941497.309780:0:14596:0:(tgt_main.c:756:tgt_ses_key_init()) slab-alloced 'session': 200 at ffff988d203b90c8. 00008000:00000010:3.0:1773941497.309790:0:14596:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff988d209199e8. 00000020:00000010:3.0:1773941497.309794:0:14596:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d0335dc00. 00000020:00000010:3.0:1773941497.309796:0:14596:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d1817c400. 00000020:00000010:3.0:1773941497.309800:0:14596:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d183507e0. 00008000:00000010:3.0:1773941497.309804:0:14596:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d107f2de8. 00020000:00000010:3.0:1773941497.309808:0:14596:0:(lov_dev.c:599:lov_device_alloc()) kmalloced '(ld)': 632 at ffff988d054e1000. 00000020:00000010:3.0:1773941497.309830:0:14596:0:(lu_tgt_pool.c:52:lu_tgt_pool_init()) kmalloced '(op->op_array)': 8 at ffff988d08b172c0. 00020000:00000010:3.0:1773941497.310011:0:14596:0:(lov_dev.c:616:lov_device_alloc()) kmalloced '(ld->ld_md_tgts)': 4096 at ffff988d02d6e000. 00020000:00000010:3.0:1773941497.310017:0:14596:0:(lov_dev.c:623:lov_device_alloc()) kmalloced '(ld->ld_lov->lov_mdc_tgts)': 4096 at ffff988d02d68000. 00000020:00000010:3.0:1773941497.310055:0:14596:0:(lu_object.c:1154:lu_site_init()) kmalloced '(s->ls_bkts)': 18432 at ffff988d05408000. 00000020:00000010:3.0:1773941497.310064:0:14596:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff988d065b0c00. 00000020:00000010:3.0:1773941497.310067:0:14596:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 192 at ffff988d02eb6600. 00000020:00000010:3.0:1773941497.310075:0:14596:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d1817c400. 00000020:00000010:3.0:1773941497.310080:0:14596:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d183507e0. 00008000:00000010:3.0:1773941497.310084:0:14596:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d107f2de8. 00000020:00000010:3.0:1773941497.310088:0:14596:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0335dc00. 00000020:00000010:3.0:1773941497.310093:0:14596:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at ffff988d03580580. 00000020:00000010:3.0:1773941497.310097:0:14596:0:(tgt_main.c:768:tgt_ses_key_fini()) slab-freed '(session)': 200 at ffff988d203b90c8. 00008000:00000010:3.0:1773941497.310101:0:14596:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff988d209199e8. 00000020:00000010:3.0:1773941497.310104:0:14596:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0335d000. 00000020:00000010:3.0:1773941497.310107:0:14596:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 160 at ffff988d02eb69c0. 00000020:00000010:3.0:1773941497.310110:0:14596:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 34 at ffff988d02f4d080. 00000020:00000010:3.0:1773941497.310116:0:14596:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at ffff988d02cba300. 00000020:00000010:3.0:1773941497.310119:0:14596:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at ffff988d02cba300. 00000020:00000010:3.0:1773941497.310124:0:14596:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at ffff988d02cba300. 00000020:00000010:3.0:1773941497.310127:0:14596:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at ffff988d02cba300. 00000020:00000010:3.0:1773941497.310131:0:14596:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 34 at ffff988d02f4d080. 00000020:00000010:3.0:1773941497.310135:0:14596:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 128 at ffff988d03580580. 00000020:00000010:3.0:1773941497.310142:0:14596:0:(genops.c:51:obd_device_alloc()) slab-alloced 'obd': 6584 at ffff988d089a8000. 00000020:00000010:3.0:1773941497.310148:0:14596:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff988d035c3000. 00000020:00000010:3.0:1773941497.310156:0:14596:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 128 at ffff988d03580580. 00000020:00000010:3.0:1773941497.310164:0:14596:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 34 at ffff988d02f4d080. 00000020:00000010:3.0:1773941497.310168:0:14596:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 34 at ffff988d02f4d080. 00000020:00000010:3.0:1773941497.310171:0:14596:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 160 at ffff988d02eb69c0. 00000001:00000010:3.0:1773941497.310180:0:14596:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at ffff988d02cba300. 00000001:00000010:3.0:1773941497.310183:0:14596:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at ffff988d107cf360. 00000001:00000010:3.0:1773941497.310185:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff988d065c5400. 00000001:00000010:3.0:1773941497.310188:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff988d0335d000. 00000001:00000010:3.0:1773941497.310191:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff988d0335dc00. 00000001:00000010:3.0:1773941497.310193:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff988d1817c400. 00000020:00000010:3.0:1773941497.310196:0:14596:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d1a235e00. 00000020:00000010:3.0:1773941497.310199:0:14596:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at ffff988d03580580. 00000020:00000010:3.0:1773941497.310201:0:14596:0:(tgt_main.c:756:tgt_ses_key_init()) slab-alloced 'session': 200 at ffff988d203b90c8. 00020000:00000010:3.0:1773941497.310205:0:14596:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff988d113f5d40. 00008000:00000010:3.0:1773941497.310208:0:14596:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff988d209199e8. 00000020:00000010:3.0:1773941497.310219:0:14596:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d04ce5c00. 00000020:00000010:3.0:1773941497.310221:0:14596:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d04ce5e00. 00000040:00000010:3.0:1773941497.310224:0:14596:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d04ce5a00. 00000020:00000010:3.0:1773941497.310227:0:14596:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d035c4000. 00000020:00000010:3.0:1773941497.310229:0:14596:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d04ce4000. 00040000:00000010:3.0:1773941497.310231:0:14596:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d02cba200. 00040000:00000010:3.0:1773941497.310234:0:14596:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d04ce5200. 00800000:00000010:3.0:1773941497.310238:0:14596:0:(lmv_obd.c:1219:lmv_device_alloc()) kmalloced '(lu)': 64 at ffff988d02f4da00. 00000020:00000010:3.0:1773941497.310364:0:14596:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff988d065b1800. 00000020:00000010:3.0:1773941497.310368:0:14596:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 512 at ffff988d04ce4400. 80000000:00000010:3.0:1773941497.310386:0:14596:0:(fld_cache.c:47:fld_cache_init()) kmalloced '(cache)': 168 at ffff988d02eb6300. 00800000:00000010:3.0:1773941497.310424:0:14596:0:(lmv_obd.c:1279:lmv_device_alloc()) kmalloced '(pat)': 280 at ffff988d04ce5400. 00800000:00000010:3.0:1773941497.310427:0:14596:0:(lmv_obd.c:1288:lmv_device_alloc()) kmalloced '(pat)': 280 at ffff988d04ce5000. 00000020:00000010:3.0:1773941497.310430:0:14596:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d04ce5e00. 00000040:00000010:3.0:1773941497.310433:0:14596:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d04ce5a00. 00000020:00000010:3.0:1773941497.310435:0:14596:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d035c4000. 00000020:00000010:3.0:1773941497.310440:0:14596:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d04ce4000. 00040000:00000010:3.0:1773941497.310445:0:14596:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d02cba200. 00040000:00000010:3.0:1773941497.310452:0:14596:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d04ce5200. 00000020:00000010:3.0:1773941497.310455:0:14596:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d04ce5c00. 00000020:00000010:3.0:1773941497.310457:0:14596:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at ffff988d03580580. 00000020:00000010:3.0:1773941497.310460:0:14596:0:(tgt_main.c:768:tgt_ses_key_fini()) slab-freed '(session)': 200 at ffff988d203b90c8. 00020000:00000010:3.0:1773941497.310463:0:14596:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff988d113f5d40. 00008000:00000010:3.0:1773941497.310466:0:14596:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff988d209199e8. 00000020:00000010:3.0:1773941497.310468:0:14596:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d1a235e00. 00000020:00000010:3.0:1773941497.310472:0:14596:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 160 at ffff988d02eb69c0. 00000020:00000010:3.0:1773941497.310473:0:14596:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 34 at ffff988d02f4d080. 00000020:00000010:3.0:1773941497.310477:0:14596:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at ffff988d02cba200. 00000020:00000010:3.0:1773941497.310479:0:14596:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at ffff988d02cba200. 00000020:00000010:3.0:1773941497.310481:0:14596:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at ffff988d02cba200. 00000020:00000010:3.0:1773941497.310483:0:14596:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at ffff988d02cba200. 00000020:00000010:3.0:1773941497.310485:0:14596:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 96 at ffff988d183507e0. 00000020:00000010:3.0:1773941497.310488:0:14596:0:(obd_config.c:1186:class_add_profile()) kmalloced '(lprof)': 48 at ffff988d02f4d080. 00000020:00000010:3.0:1773941497.310491:0:14596:0:(obd_config.c:1192:class_add_profile()) kmalloced '(lprof->lp_profile)': 14 at ffff988d03879680. 00000020:00000010:3.0:1773941497.310493:0:14596:0:(obd_config.c:1198:class_add_profile()) kmalloced '(lprof->lp_dt)': 14 at ffff988d03879cd0. 00000020:00000010:3.0:1773941497.310495:0:14596:0:(obd_config.c:1205:class_add_profile()) kmalloced '(lprof->lp_md)': 14 at ffff988d03879540. 00000020:00000010:3.0:1773941497.310496:0:14596:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 96 at ffff988d183507e0. 00000020:00000010:3.0:1773941497.310498:0:14596:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at ffff988d02cba200. 00000020:00000010:3.0:1773941497.310500:0:14596:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at ffff988d02cba200. 00000020:00000010:3.0:1773941497.310503:0:14596:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at ffff988d02cba200. 00000020:00000010:3.0:1773941497.310504:0:14596:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at ffff988d02cba200. 00000020:00000010:3.0:1773941497.310506:0:14596:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 64 at ffff988d02f4d840. 00000100:00000010:3.0:1773941497.310510:0:14596:0:(lustre_peer.c:71:class_add_uuid()) kmalloced '(data)': 704 at ffff988d065b1000. 00000400:00000200:3.0:1773941497.310514:0:14596:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:3.0:1773941497.310518:0:14596:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.204.114@tcp 00000400:00000200:3.0:1773941497.310521:0:14596:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.310524:0:14596:0:(peer.c:1737:lnet_peer_add()) peer 192.168.204.114@tcp NID flags 0x100001: -17 00000400:00000200:3.0:1773941497.310527:0:14596:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.310532:0:14596:0:(peer.c:2767:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1773941497.310535:0:14596:0:(peer.c:2776:lnet_discover_peer_locked()) peer 192.168.204.114@tcp NID 192.168.204.114@tcp: 0. pending discovery 00000400:00000200:3.0:1773941497.310543:0:14596:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.204.114@tcp 00000100:00000010:3.0:1773941497.310546:0:14596:0:(lustre_peer.c:113:class_add_uuid()) kfreed 'data': 704 at ffff988d065b1000. 00000020:00000010:3.0:1773941497.310549:0:14596:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 64 at ffff988d02f4d840. 00000020:00000010:3.0:1773941497.310552:0:14596:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 39 at ffff988d02f4d840. 00000020:00000010:3.0:1773941497.310555:0:14596:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 136 at ffff988d02eb69c0. 00000020:00000010:3.0:1773941497.310563:0:14596:0:(genops.c:51:obd_device_alloc()) slab-alloced 'obd': 6584 at ffff988d089ab370. 00000020:00000010:3.0:1773941497.310567:0:14596:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff988d035c4000. 00000020:00000010:3.0:1773941497.310574:0:14596:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 136 at ffff988d02eb69c0. 00000020:00000010:3.0:1773941497.310577:0:14596:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 39 at ffff988d02f4d840. 00000020:00000010:3.0:1773941497.310581:0:14596:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 39 at ffff988d02f4d840. 00000020:00000010:3.0:1773941497.310585:0:14596:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 136 at ffff988d02eb69c0. 00000001:00000010:3.0:1773941497.310608:0:14596:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at ffff988d02cba200. 00000001:00000010:3.0:1773941497.310614:0:14596:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at ffff988d107cf7e0. 00000001:00000010:3.0:1773941497.310616:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff988d04ce5c00. 00000001:00000010:3.0:1773941497.310621:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff988d04ce5200. 00000001:00000010:3.0:1773941497.310625:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff988d04ce4000. 00000001:00000010:3.0:1773941497.310629:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff988d04ce5a00. 00000020:00000010:3.0:1773941497.310633:0:14596:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d04ce5e00. 00000020:00000010:3.0:1773941497.310640:0:14596:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at ffff988d03580580. 00000020:00000010:3.0:1773941497.310644:0:14596:0:(tgt_main.c:756:tgt_ses_key_init()) slab-alloced 'session': 200 at ffff988d203b90c8. 00020000:00000010:3.0:1773941497.310648:0:14596:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff988d113f5d40. 00008000:00000010:3.0:1773941497.310653:0:14596:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff988d209199e8. 00000020:00000010:3.0:1773941497.310659:0:14596:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d04ce4c00. 00000020:00000010:3.0:1773941497.310663:0:14596:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d04ce4600. 00000020:00000010:3.0:1773941497.310666:0:14596:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d183507e0. 00020000:00000010:3.0:1773941497.310671:0:14596:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff988d20218840. 00008000:00000010:3.0:1773941497.310673:0:14596:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d107f2de8. 00000002:00000010:3.0:1773941497.310678:0:14596:0:(mdc_dev.c:1726:mdc_device_alloc()) kmalloced '(osc)': 104 at ffff988d03580080. 00010000:00000010:3.0:1773941497.310690:0:14596:0:(ldlm_lib.c:421:client_obd_setup()) kmalloced '(cli->cl_read_io_latency_by_size)': 3920 at ffff988d02d6d000. 00010000:00000010:3.0:1773941497.310701:0:14596:0:(ldlm_lib.c:434:client_obd_setup()) kmalloced '(cli->cl_write_io_latency_by_size)': 3920 at ffff988d02d69000. 00010000:00000010:3.0:1773941497.310706:0:14596:0:(ldlm_lib.c:522:client_obd_setup()) kmalloced '(cli->cl_mod_tag_bitmap)': 64 at ffff988d02f4d680. 00000020:00000010:3.0:1773941497.310717:0:14596:0:(genops.c:1283:class_new_import()) kmalloced '(imp)': 1712 at ffff988d035c7000. 00000100:00000200:3.0:1773941497.310734:0:14596:0:(events.c:560:ptlrpc_uuid_to_peer()) 192.168.204.114@tcp->12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941497.310737:0:14596:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.310741:0:14596:0:(peer.c:1471:LNetPrimaryNID()) NID 192.168.204.114@tcp primary NID 192.168.204.114@tcp rc 0 00010000:00000010:3.0:1773941497.310746:0:14596:0:(ldlm_lib.c:70:import_set_conn()) kmalloced '(imp_conn)': 88 at ffff988d18350de0. 00010000:00000010:3.0:1773941497.310751:0:14596:0:(ldlm_resource.c:955:ldlm_namespace_new()) kmalloced '(ns)': 848 at ffff988d203fe000. 00000001:00000010:3.0:1773941497.310754:0:14596:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 280 at ffff988d04ce4e00. 00000001:00000010:3.0:1773941497.310758:0:14596:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 256 at ffff988d02cbae00. 00000001:00000010:3.0:1773941497.310817:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81cb1000. 00000001:00000010:3.0:1773941497.310870:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81cbb000. 00000001:00000010:3.0:1773941497.310927:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81cc5000. 00000001:00000010:3.0:1773941497.311030:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81cd9000. 00000001:00000010:3.0:1773941497.311084:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81ce3000. 00000001:00000010:3.0:1773941497.311154:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81cf9000. 00000001:00000010:3.0:1773941497.311205:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81d03000. 00000001:00000010:3.0:1773941497.311271:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81d0d000. 00000001:00000010:3.0:1773941497.311325:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81d17000. 00000001:00000010:3.0:1773941497.311388:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81d21000. 00000001:00000010:3.0:1773941497.311438:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81d2b000. 00000001:00000010:3.0:1773941497.311486:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81d35000. 00000001:00000010:3.0:1773941497.311532:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81d3f000. 00000001:00000010:3.0:1773941497.311579:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81d51000. 00000001:00000010:3.0:1773941497.311627:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81d5b000. 00000001:00000010:3.0:1773941497.311693:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81d65000. 00000001:00000010:3.0:1773941497.311747:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81d79000. 00000001:00000010:3.0:1773941497.311804:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81d91000. 00000001:00000010:3.0:1773941497.311851:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81d9b000. 00000001:00000010:3.0:1773941497.311903:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81da5000. 00000001:00000010:3.0:1773941497.311954:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81daf000. 00000001:00000010:3.0:1773941497.312086:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81db9000. 00000001:00000010:3.0:1773941497.312160:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81dc3000. 00000001:00000010:3.0:1773941497.312224:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81e01000. 00000001:00000010:3.0:1773941497.312292:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81e0b000. 00000001:00000010:3.0:1773941497.312357:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81e15000. 00000001:00000010:3.0:1773941497.312433:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81e1f000. 00000001:00000010:3.0:1773941497.312501:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81e29000. 00000001:00000010:3.0:1773941497.312558:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81e33000. 00000001:00000010:3.0:1773941497.312616:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81e3d000. 00000001:00000010:3.0:1773941497.312695:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81e47000. 00000001:00000010:3.0:1773941497.312763:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81e51000. 00000020:00000010:3.0:1773941497.312895:0:14596:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff988d203fd400. 00000020:00000010:3.0:1773941497.312899:0:14596:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 96 at ffff988d18302960. 00000020:00000010:3.0:1773941497.313054:0:14596:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff988d203fdc00. 00000020:00000010:3.0:1773941497.313058:0:14596:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 352 at ffff988d04ce4200. 00000020:00000010:3.0:1773941497.313287:0:14596:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff988d203ff000. 00000020:00000010:3.0:1773941497.313290:0:14596:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 512 at ffff988d04ce4800. 00000020:00000010:3.0:1773941497.313311:0:14596:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff988d203ff800. 00000020:00000010:3.0:1773941497.313314:0:14596:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 3680 at ffff988d02d6b000. 00000040:00000010:3.0:1773941497.313335:0:14596:0:(llog_obd.c:26:llog_new_ctxt()) kmalloced '(ctxt)': 168 at ffff988d02eb6e40. 00000002:00000010:3.0:1773941497.313343:0:14596:0:(mdc_changelog.c:816:mdc_changelog_cdev_init()) kmalloced '(entry)': 1304 at ffff988d035c5800. 00000020:00000010:3.0:1773941497.313534:0:14596:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d04ce4600. 00000020:00000010:3.0:1773941497.313538:0:14596:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d183507e0. 00020000:00000010:3.0:1773941497.313543:0:14596:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff988d20218840. 00008000:00000010:3.0:1773941497.313548:0:14596:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d107f2de8. 00000020:00000010:3.0:1773941497.313554:0:14596:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d04ce4c00. 00000020:00000010:3.0:1773941497.313557:0:14596:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at ffff988d03580580. 00000020:00000010:3.0:1773941497.313559:0:14596:0:(tgt_main.c:768:tgt_ses_key_fini()) slab-freed '(session)': 200 at ffff988d203b90c8. 00020000:00000010:3.0:1773941497.313561:0:14596:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff988d113f5d40. 00008000:00000010:3.0:1773941497.313563:0:14596:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff988d209199e8. 00000020:00000010:3.0:1773941497.313565:0:14596:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d04ce5e00. 00000020:00000010:3.0:1773941497.313567:0:14596:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 136 at ffff988d02eb69c0. 00000020:00000010:3.0:1773941497.313569:0:14596:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 39 at ffff988d02f4d840. 00000020:00000010:3.0:1773941497.313578:0:14596:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 34 at ffff988d08249fc0. 00000020:00000010:3.0:1773941497.313582:0:14596:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 152 at ffff988d02eb69c0. 00000020:00000010:3.0:1773941497.313586:0:14596:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d04ce5e00. 00000020:00000010:3.0:1773941497.313589:0:14596:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d04ce4c00. 00000040:00000010:3.0:1773941497.313591:0:14596:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d04ce4600. 00000020:00000010:3.0:1773941497.313593:0:14596:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d035c7800. 00000020:00000010:3.0:1773941497.313596:0:14596:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d04ce4a00. 00040000:00000010:3.0:1773941497.313599:0:14596:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d21fadf00. 00040000:00000010:3.0:1773941497.313602:0:14596:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d04ce5800. 00800000:00000010:3.0:1773941497.313613:0:14596:0:(lmv_obd.c:433:lmv_add_target()) kmalloced '(tgt)': 368 at ffff988d04ce5600. 00020000:00000010:3.0:1773941497.313617:0:14596:0:(lu_tgt_descs.c:371:ltd_add_tgt()) kmalloced '(ltd->ltd_tgt_idx[index / (((1UL) << 12) / sizeof(void *))])': 4096 at ffff988d07e47000. 00000020:00000010:3.0:1773941497.313620:0:14596:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d04ce4c00. 00000040:00000010:3.0:1773941497.313625:0:14596:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d04ce4600. 00000020:00000010:3.0:1773941497.313628:0:14596:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d035c7800. 00000020:00000010:3.0:1773941497.313630:0:14596:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d04ce4a00. 00040000:00000010:3.0:1773941497.313632:0:14596:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d21fadf00. 00040000:00000010:3.0:1773941497.313634:0:14596:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d04ce5800. 00000020:00000010:3.0:1773941497.313640:0:14596:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d04ce5e00. 00000020:00000010:3.0:1773941497.313647:0:14596:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d04ce5e00. 00000020:00000010:3.0:1773941497.313651:0:14596:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d04ce5800. 00000020:00000010:3.0:1773941497.313655:0:14596:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d183021e0. 00020000:00000010:3.0:1773941497.313659:0:14596:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff988d20218840. 00008000:00000010:3.0:1773941497.313664:0:14596:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d107f2de8. 00000020:00000010:3.0:1773941497.313676:0:14596:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d04ce5800. 00000020:00000010:3.0:1773941497.313680:0:14596:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d183021e0. 00020000:00000010:3.0:1773941497.313686:0:14596:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff988d20218840. 00008000:00000010:3.0:1773941497.313690:0:14596:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d107f2de8. 00000020:00000010:3.0:1773941497.313699:0:14596:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d04ce5e00. 00000020:00000010:3.0:1773941497.313703:0:14596:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 152 at ffff988d02eb69c0. 00000020:00000010:3.0:1773941497.313707:0:14596:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 34 at ffff988d08249fc0. 00000020:00000010:3.0:1773941497.313715:0:14596:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at ffff988d21fadf00. 00000020:00000010:3.0:1773941497.313718:0:14596:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at ffff988d21fadf00. 00000020:00000010:3.0:1773941497.313731:0:14596:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at ffff988d21fadf00. 00000020:00000010:3.0:1773941497.313734:0:14596:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at ffff988d21fadf00. 00000020:00000010:3.0:1773941497.313741:0:14596:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at ffff988d21fadf00. 00000020:00000010:3.0:1773941497.313743:0:14596:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at ffff988d21fadf00. 00000020:00000010:3.0:1773941497.313748:0:14596:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 64 at ffff988d08249fc0. 00000100:00000010:3.0:1773941497.313752:0:14596:0:(lustre_peer.c:71:class_add_uuid()) kmalloced '(data)': 704 at ffff988d203fe800. 00000400:00000200:3.0:1773941497.313755:0:14596:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:3.0:1773941497.313759:0:14596:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.204.114@tcp 00000400:00000200:3.0:1773941497.313763:0:14596:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.313765:0:14596:0:(peer.c:1737:lnet_peer_add()) peer 192.168.204.114@tcp NID flags 0x100001: -17 00000400:00000200:3.0:1773941497.313768:0:14596:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.313771:0:14596:0:(peer.c:2767:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1773941497.313774:0:14596:0:(peer.c:2776:lnet_discover_peer_locked()) peer 192.168.204.114@tcp NID 192.168.204.114@tcp: 0. pending discovery 00000400:00000200:3.0:1773941497.313776:0:14596:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.204.114@tcp 00000100:00000010:3.0:1773941497.313777:0:14596:0:(lustre_peer.c:113:class_add_uuid()) kfreed 'data': 704 at ffff988d203fe800. 00000020:00000010:3.0:1773941497.313780:0:14596:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 64 at ffff988d08249fc0. 00000020:00000010:3.0:1773941497.313785:0:14596:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 39 at ffff988d08249fc0. 00000020:00000010:3.0:1773941497.313788:0:14596:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 136 at ffff988d02eb69c0. 00000020:00000010:3.0:1773941497.313795:0:14596:0:(genops.c:51:obd_device_alloc()) slab-alloced 'obd': 6584 at ffff988d089acd28. 00000020:00000010:3.0:1773941497.313799:0:14596:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff988d035c7800. 00000020:00000010:3.0:1773941497.313814:0:14596:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 136 at ffff988d02eb69c0. 00000020:00000010:3.0:1773941497.313819:0:14596:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 39 at ffff988d08249fc0. 00000020:00000010:3.0:1773941497.313824:0:14596:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 39 at ffff988d08249fc0. 00000020:00000010:3.0:1773941497.313827:0:14596:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 136 at ffff988d02eb69c0. 00000001:00000010:3.0:1773941497.313849:0:14596:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at ffff988d21fadf00. 00000001:00000010:3.0:1773941497.313858:0:14596:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at ffff988d107cfaa0. 00000001:00000010:3.0:1773941497.313862:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff988d04ce5e00. 00000001:00000010:3.0:1773941497.313866:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff988d04ce5800. 00000001:00000010:3.0:1773941497.313869:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff988d04ce4a00. 00000001:00000010:3.0:1773941497.313872:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff988d04ce4600. 00000020:00000010:3.0:1773941497.313876:0:14596:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d04ce4c00. 00000020:00000010:3.0:1773941497.313880:0:14596:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at ffff988d03580580. 00000020:00000010:3.0:1773941497.313891:0:14596:0:(tgt_main.c:756:tgt_ses_key_init()) slab-alloced 'session': 200 at ffff988d203b90c8. 00020000:00000010:3.0:1773941497.313896:0:14596:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff988d113f5d40. 00008000:00000010:3.0:1773941497.313901:0:14596:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff988d209199e8. 00000020:00000010:3.0:1773941497.313908:0:14596:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d1a235e00. 00000020:00000010:3.0:1773941497.313925:0:14596:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d065dd800. 00000020:00000010:3.0:1773941497.313932:0:14596:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d183021e0. 00020000:00000010:3.0:1773941497.313936:0:14596:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff988d20218840. 00008000:00000010:3.0:1773941497.313940:0:14596:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d107f2de8. 00000002:00000010:3.0:1773941497.313945:0:14596:0:(mdc_dev.c:1726:mdc_device_alloc()) kmalloced '(osc)': 104 at ffff988d03580480. 00010000:00000010:3.0:1773941497.313955:0:14596:0:(ldlm_lib.c:421:client_obd_setup()) kmalloced '(cli->cl_read_io_latency_by_size)': 3920 at ffff988d07e43000. 00010000:00000010:3.0:1773941497.313996:0:14596:0:(ldlm_lib.c:434:client_obd_setup()) kmalloced '(cli->cl_write_io_latency_by_size)': 3920 at ffff988d02ecf000. 00010000:00000010:3.0:1773941497.314007:0:14596:0:(ldlm_lib.c:522:client_obd_setup()) kmalloced '(cli->cl_mod_tag_bitmap)': 64 at ffff988d082498c0. 00000020:00000010:3.0:1773941497.314013:0:14596:0:(genops.c:1283:class_new_import()) kmalloced '(imp)': 1712 at ffff988d035c6000. 00000100:00000200:3.0:1773941497.314032:0:14596:0:(events.c:560:ptlrpc_uuid_to_peer()) 192.168.204.114@tcp->12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941497.314039:0:14596:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.314047:0:14596:0:(peer.c:1471:LNetPrimaryNID()) NID 192.168.204.114@tcp primary NID 192.168.204.114@tcp rc 0 00010000:00000010:3.0:1773941497.314051:0:14596:0:(ldlm_lib.c:70:import_set_conn()) kmalloced '(imp_conn)': 88 at ffff988d18302a80. 00010000:00000010:3.0:1773941497.314054:0:14596:0:(ldlm_resource.c:955:ldlm_namespace_new()) kmalloced '(ns)': 848 at ffff988d203fd800. 00000001:00000010:3.0:1773941497.314056:0:14596:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 280 at ffff988d065dc800. 00000001:00000010:3.0:1773941497.314059:0:14596:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 256 at ffff988d21fad100. 00000001:00000010:3.0:1773941497.314104:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81e5b000. 00000001:00000010:3.0:1773941497.314167:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81e65000. 00000001:00000010:3.0:1773941497.314229:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81e6f000. 00000001:00000010:3.0:1773941497.314298:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81e79000. 00000001:00000010:3.0:1773941497.314356:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81e83000. 00000001:00000010:3.0:1773941497.314403:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81e8d000. 00000001:00000010:3.0:1773941497.314450:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81e97000. 00000001:00000010:3.0:1773941497.314497:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81ea1000. 00000001:00000010:3.0:1773941497.314540:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81eb1000. 00000001:00000010:3.0:1773941497.314590:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81ec1000. 00000001:00000010:3.0:1773941497.314634:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81ed1000. 00000001:00000010:3.0:1773941497.314685:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81edb000. 00000001:00000010:3.0:1773941497.314728:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81ee5000. 00000001:00000010:3.0:1773941497.314773:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81eef000. 00000001:00000010:3.0:1773941497.314816:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81ef9000. 00000001:00000010:3.0:1773941497.314860:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81f03000. 00000001:00000010:3.0:1773941497.314903:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81f0d000. 00000001:00000010:3.0:1773941497.314949:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81f17000. 00000001:00000010:3.0:1773941497.315049:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81f21000. 00000001:00000010:3.0:1773941497.315104:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81f2b000. 00000001:00000010:3.0:1773941497.315163:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81f35000. 00000001:00000010:3.0:1773941497.315209:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81f3f000. 00000001:00000010:3.0:1773941497.315284:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81f49000. 00000001:00000010:3.0:1773941497.315334:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81f53000. 00000001:00000010:3.0:1773941497.315391:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81f5d000. 00000001:00000010:3.0:1773941497.315447:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81f67000. 00000001:00000010:3.0:1773941497.315498:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81f71000. 00000001:00000010:3.0:1773941497.315561:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81f7b000. 00000001:00000010:3.0:1773941497.315613:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81f85000. 00000001:00000010:3.0:1773941497.315654:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81f8f000. 00000001:00000010:3.0:1773941497.315702:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81f99000. 00000001:00000010:3.0:1773941497.315741:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 32944 at ffffb6ec81fa3000. 00000020:00000010:3.0:1773941497.315794:0:14596:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff988d203fd000. 00000020:00000010:3.0:1773941497.315797:0:14596:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 96 at ffff988d20936060. 00000020:00000010:3.0:1773941497.315846:0:14596:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff988d203ffc00. 00000020:00000010:3.0:1773941497.315852:0:14596:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 352 at ffff988d065dca00. 00000020:00000010:3.0:1773941497.316036:0:14596:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff988d203fcc00. 00000020:00000010:3.0:1773941497.316039:0:14596:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 512 at ffff988d065dde00. 00000020:00000010:3.0:1773941497.316073:0:14596:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff988d111d3800. 00000020:00000010:3.0:1773941497.316078:0:14596:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 3680 at ffff988d02ec9000. 00000040:00000010:3.0:1773941497.316104:0:14596:0:(llog_obd.c:26:llog_new_ctxt()) kmalloced '(ctxt)': 168 at ffff988d02eb6540. 00000002:00000010:3.0:1773941497.316111:0:14596:0:(mdc_changelog.c:816:mdc_changelog_cdev_init()) kmalloced '(entry)': 1304 at ffff988d035c3800. 00000020:00000010:3.0:1773941497.316222:0:14596:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d065dd800. 00000020:00000010:3.0:1773941497.316225:0:14596:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d183021e0. 00020000:00000010:3.0:1773941497.316228:0:14596:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff988d20218840. 00008000:00000010:3.0:1773941497.316231:0:14596:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d107f2de8. 00000020:00000010:3.0:1773941497.316233:0:14596:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d1a235e00. 00000020:00000010:3.0:1773941497.316236:0:14596:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at ffff988d03580580. 00000020:00000010:3.0:1773941497.316238:0:14596:0:(tgt_main.c:768:tgt_ses_key_fini()) slab-freed '(session)': 200 at ffff988d203b90c8. 00020000:00000010:3.0:1773941497.316241:0:14596:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff988d113f5d40. 00008000:00000010:3.0:1773941497.316256:0:14596:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff988d209199e8. 00000020:00000010:3.0:1773941497.316258:0:14596:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d04ce4c00. 00000020:00000010:3.0:1773941497.316261:0:14596:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 136 at ffff988d02eb69c0. 00000020:00000010:3.0:1773941497.316263:0:14596:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 39 at ffff988d08249fc0. 00000020:00000010:3.0:1773941497.316269:0:14596:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 34 at ffff988d082b2f80. 00000020:00000010:3.0:1773941497.316273:0:14596:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 152 at ffff988d02eb69c0. 00000020:00000010:3.0:1773941497.316278:0:14596:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d065dd800. 00000020:00000010:3.0:1773941497.316280:0:14596:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d065dda00. 00000040:00000010:3.0:1773941497.316283:0:14596:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d065dc000. 00000020:00000010:3.0:1773941497.316286:0:14596:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d035c2800. 00000020:00000010:3.0:1773941497.316289:0:14596:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d065dd200. 00040000:00000010:3.0:1773941497.316292:0:14596:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d21fadc00. 00040000:00000010:3.0:1773941497.316295:0:14596:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d065dd600. 00800000:00000010:3.0:1773941497.316305:0:14596:0:(lmv_obd.c:433:lmv_add_target()) kmalloced '(tgt)': 368 at ffff988d065dc400. 00000020:00000010:3.0:1773941497.316308:0:14596:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d065dda00. 00000040:00000010:3.0:1773941497.316311:0:14596:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d065dc000. 00000020:00000010:3.0:1773941497.316315:0:14596:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d035c2800. 00000020:00000010:3.0:1773941497.316318:0:14596:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d065dd200. 00040000:00000010:3.0:1773941497.316322:0:14596:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d21fadc00. 00040000:00000010:3.0:1773941497.316324:0:14596:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d065dd600. 00000020:00000010:3.0:1773941497.316327:0:14596:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d065dd800. 00000020:00000010:3.0:1773941497.316332:0:14596:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d065dd800. 00000020:00000010:3.0:1773941497.316335:0:14596:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d065dd600. 00000020:00000010:3.0:1773941497.316340:0:14596:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d20936240. 00020000:00000010:3.0:1773941497.316344:0:14596:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff988d20218840. 00008000:00000010:3.0:1773941497.316355:0:14596:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d107f2de8. 00000020:00000010:3.0:1773941497.316365:0:14596:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d065dd600. 00000020:00000010:3.0:1773941497.316369:0:14596:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d20936240. 00020000:00000010:3.0:1773941497.316374:0:14596:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff988d20218840. 00008000:00000010:3.0:1773941497.316379:0:14596:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d107f2de8. 00000020:00000010:3.0:1773941497.316386:0:14596:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d065dd800. 00000020:00000010:3.0:1773941497.316390:0:14596:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 152 at ffff988d02eb69c0. 00000020:00000010:3.0:1773941497.316396:0:14596:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 34 at ffff988d082b2f80. 00000020:00000010:3.0:1773941497.316405:0:14596:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at ffff988d21fadc00. 00000020:00000010:3.0:1773941497.316408:0:14596:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at ffff988d21fadc00. 00000020:00000010:3.0:1773941497.316414:0:14596:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at ffff988d21fadc00. 00000020:00000010:3.0:1773941497.316417:0:14596:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at ffff988d21fadc00. 00000020:00000010:3.0:1773941497.316424:0:14596:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at ffff988d21fadc00. 00000020:00000010:3.0:1773941497.316426:0:14596:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at ffff988d21fadc00. 00000020:00000010:3.0:1773941497.316433:0:14596:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 64 at ffff988d082b2f80. 00000100:00000010:3.0:1773941497.316437:0:14596:0:(lustre_peer.c:71:class_add_uuid()) kmalloced '(data)': 704 at ffff988d111d2800. 00000400:00000200:3.0:1773941497.316442:0:14596:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:3.0:1773941497.316445:0:14596:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.204.114@tcp 00000400:00000200:3.0:1773941497.316450:0:14596:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.316454:0:14596:0:(peer.c:1737:lnet_peer_add()) peer 192.168.204.114@tcp NID flags 0x100001: -17 00000400:00000200:3.0:1773941497.316459:0:14596:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.316469:0:14596:0:(peer.c:2767:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1773941497.316474:0:14596:0:(peer.c:2776:lnet_discover_peer_locked()) peer 192.168.204.114@tcp NID 192.168.204.114@tcp: 0. pending discovery 00000400:00000200:3.0:1773941497.316478:0:14596:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.204.114@tcp 00000100:00000010:3.0:1773941497.316480:0:14596:0:(lustre_peer.c:113:class_add_uuid()) kfreed 'data': 704 at ffff988d111d2800. 00000020:00000010:3.0:1773941497.316484:0:14596:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 64 at ffff988d082b2f80. 00000020:00000010:3.0:1773941497.316489:0:14596:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 39 at ffff988d082b2f80. 00000020:00000010:3.0:1773941497.316493:0:14596:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 136 at ffff988d02eb69c0. 00000020:00000010:3.0:1773941497.316511:0:14596:0:(genops.c:51:obd_device_alloc()) slab-alloced 'obd': 6584 at ffff988d06460000. 00000020:00000010:3.0:1773941497.316516:0:14596:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff988d035c2800. 00000020:00000010:3.0:1773941497.316532:0:14596:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 136 at ffff988d02eb69c0. 00000020:00000010:3.0:1773941497.316535:0:14596:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 39 at ffff988d082b2f80. 00000020:00000010:3.0:1773941497.316544:0:14596:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 39 at ffff988d082b2f80. 00000020:00000010:3.0:1773941497.316547:0:14596:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 136 at ffff988d02eb69c0. 00000001:00000010:3.0:1773941497.316570:0:14596:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at ffff988d21fadc00. 00000001:00000010:3.0:1773941497.316574:0:14596:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at ffff988d09246740. 00000001:00000010:3.0:1773941497.316576:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff988d065dd800. 00000001:00000010:3.0:1773941497.316579:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff988d065dd600. 00000001:00000010:3.0:1773941497.316582:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff988d065dd200. 00000001:00000010:3.0:1773941497.316586:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff988d065dc000. 00000020:00000010:3.0:1773941497.316590:0:14596:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d065dda00. 00000020:00000010:3.0:1773941497.316593:0:14596:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at ffff988d03580580. 00000020:00000010:3.0:1773941497.316596:0:14596:0:(tgt_main.c:756:tgt_ses_key_init()) slab-alloced 'session': 200 at ffff988d203b90c8. 00020000:00000010:3.0:1773941497.316599:0:14596:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff988d113f5d40. 00008000:00000010:3.0:1773941497.316602:0:14596:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff988d209199e8. 00000020:00000010:3.0:1773941497.316606:0:14596:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d065dd400. 00000020:00000010:3.0:1773941497.316609:0:14596:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d065dd000. 00000020:00000010:3.0:1773941497.316614:0:14596:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d20936240. 00020000:00000010:3.0:1773941497.316616:0:14596:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff988d20218840. 00008000:00000010:3.0:1773941497.316620:0:14596:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d107f2de8. 00000008:00000010:3.0:1773941497.316627:0:14596:0:(osc_dev.c:192:osc_device_alloc()) kmalloced '(osc)': 104 at ffff988d03580b80. 00010000:00000010:3.0:1773941497.316638:0:14596:0:(ldlm_lib.c:421:client_obd_setup()) kmalloced '(cli->cl_read_io_latency_by_size)': 3920 at ffff988d02ecb000. 00010000:00000010:3.0:1773941497.316664:0:14596:0:(ldlm_lib.c:434:client_obd_setup()) kmalloced '(cli->cl_write_io_latency_by_size)': 3920 at ffff988d02eca000. 00000020:00000010:3.0:1773941497.316675:0:14596:0:(genops.c:1283:class_new_import()) kmalloced '(imp)': 1712 at ffff988d180dc800. 00000100:00000200:3.0:1773941497.316704:0:14596:0:(events.c:560:ptlrpc_uuid_to_peer()) 192.168.204.114@tcp->12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941497.316711:0:14596:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.316719:0:14596:0:(peer.c:1471:LNetPrimaryNID()) NID 192.168.204.114@tcp primary NID 192.168.204.114@tcp rc 0 00010000:00000010:3.0:1773941497.316725:0:14596:0:(ldlm_lib.c:70:import_set_conn()) kmalloced '(imp_conn)': 88 at ffff988d20936c00. 00010000:00000010:3.0:1773941497.316732:0:14596:0:(ldlm_resource.c:955:ldlm_namespace_new()) kmalloced '(ns)': 848 at ffff988d111d1800. 00000001:00000010:3.0:1773941497.316736:0:14596:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 280 at ffff988d065dcc00. 00000001:00000010:3.0:1773941497.316743:0:14596:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 128 at ffff988d03580380. 00000001:00000010:3.0:1773941497.316750:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at ffff988d20300000. 00000001:00000010:3.0:1773941497.316757:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at ffff988d20304000. 00000001:00000010:3.0:1773941497.316765:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at ffff988d03378000. 00000001:00000010:3.0:1773941497.316777:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at ffff988d0337c000. 00000001:00000010:3.0:1773941497.316783:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at ffff988d0337e000. 00000001:00000010:3.0:1773941497.316789:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at ffff988d0337a000. 00000001:00000010:3.0:1773941497.316795:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at ffff988d107fe000. 00000001:00000010:3.0:1773941497.316801:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at ffff988d107fa000. 00000001:00000010:3.0:1773941497.316807:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at ffff988d107f8000. 00000001:00000010:3.0:1773941497.316815:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at ffff988d107fc000. 00000001:00000010:3.0:1773941497.316823:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at ffff988d065ec000. 00000001:00000010:3.0:1773941497.316830:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at ffff988d065ee000. 00000001:00000010:3.0:1773941497.316837:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at ffff988d065ea000. 00000001:00000010:3.0:1773941497.316844:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at ffff988d065e8000. 00000001:00000010:3.0:1773941497.316863:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at ffff988d0646a000. 00000001:00000010:3.0:1773941497.316872:0:14596:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 4272 at ffff988d0646e000. 00000020:00000010:3.0:1773941497.316910:0:14596:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff988d111d1000. 00000020:00000010:3.0:1773941497.316912:0:14596:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 96 at ffff988d20936ba0. 00000020:00000010:3.0:1773941497.317021:0:14596:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff988d111d3c00. 00000020:00000010:3.0:1773941497.317024:0:14596:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 352 at ffff988d065dc600. 00000020:00000010:3.0:1773941497.317311:0:14596:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff988d111d0800. 00000020:00000010:3.0:1773941497.317316:0:14596:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 3680 at ffff988d02ece000. 00000020:00000010:3.0:1773941497.317338:0:14596:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d065dd000. 00000020:00000010:3.0:1773941497.317342:0:14596:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d20936240. 00020000:00000010:3.0:1773941497.317345:0:14596:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff988d20218840. 00008000:00000010:3.0:1773941497.317347:0:14596:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d107f2de8. 00000020:00000010:3.0:1773941497.317352:0:14596:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d065dd400. 00000020:00000010:3.0:1773941497.317355:0:14596:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at ffff988d03580580. 00000020:00000010:3.0:1773941497.317357:0:14596:0:(tgt_main.c:768:tgt_ses_key_fini()) slab-freed '(session)': 200 at ffff988d203b90c8. 00020000:00000010:3.0:1773941497.317360:0:14596:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff988d113f5d40. 00008000:00000010:3.0:1773941497.317363:0:14596:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff988d209199e8. 00000020:00000010:3.0:1773941497.317367:0:14596:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d065dda00. 00000020:00000010:3.0:1773941497.317372:0:14596:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 136 at ffff988d02eb69c0. 00000020:00000010:3.0:1773941497.317375:0:14596:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 39 at ffff988d082b2f80. 00000020:00000010:3.0:1773941497.317386:0:14596:0:(obd_config.c:2030:class_config_llog_handler()) kmalloced '(inst_name)': 34 at ffff988d082b2f80. 00000020:00000010:3.0:1773941497.317390:0:14596:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 120 at ffff988d03580580. 00000020:00000010:3.0:1773941497.317395:0:14596:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d065dda00. 00000020:00000010:3.0:1773941497.317398:0:14596:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d065dd400. 00000020:00000010:3.0:1773941497.317401:0:14596:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d20936240. 00000008:00000010:3.0:1773941497.317407:0:14596:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff988d113fd940. 00020000:00000010:3.0:1773941497.317410:0:14596:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff988d20218840. 00008000:00000010:3.0:1773941497.317413:0:14596:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d107f2de8. 00020000:00000010:3.0:1773941497.317424:0:14596:0:(lov_obd.c:492:lov_add_target()) kmalloced '(tgt)': 368 at ffff988d065dd000. 00020000:00000010:3.0:1773941497.317431:0:14596:0:(lu_tgt_descs.c:371:ltd_add_tgt()) kmalloced '(ltd->ltd_tgt_idx[index / (((1UL) << 12) / sizeof(void *))])': 4096 at ffff988d02ec8000. 00020000:00000010:3.0:1773941497.317439:0:14596:0:(lov_dev.c:357:lov_expand_targets()) kmalloced '(newd)': 512 at ffff988d065dc200. 00000020:00000010:3.0:1773941497.317442:0:14596:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d065dd400. 00000020:00000010:3.0:1773941497.317446:0:14596:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d20936240. 00000008:00000010:3.0:1773941497.317451:0:14596:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff988d113fd940. 00020000:00000010:3.0:1773941497.317458:0:14596:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff988d20218840. 00008000:00000010:3.0:1773941497.317461:0:14596:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d107f2de8. 00000020:00000010:3.0:1773941497.317465:0:14596:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d065dda00. 00000020:00000010:3.0:1773941497.317467:0:14596:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 120 at ffff988d03580580. 00000020:00000010:3.0:1773941497.317469:0:14596:0:(obd_config.c:2125:class_config_llog_handler()) kfreed 'inst_name': 34 at ffff988d082b2f80. 00000020:00000010:3.0:1773941497.317474:0:14596:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at ffff988d21fada00. 00000020:00000010:3.0:1773941497.317476:0:14596:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at ffff988d21fada00. 00000020:00000010:3.0:1773941497.317481:0:14596:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at ffff988d21fada00. 00000020:00000010:3.0:1773941497.317483:0:14596:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at ffff988d21fada00. 00000020:00000010:3.0:1773941497.317488:0:14596:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 56 at ffff988d082b2f80. 00000020:00000010:3.0:1773941497.317492:0:14596:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 56 at ffff988d082b2f80. 00000100:00000010:3.0:1773941497.317509:0:14596:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d111f1500. 02000000:00000010:3.0:1773941497.317527:0:14596:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065dda00. 00000100:00000010:3.0:1773941497.317545:0:14596:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb69c0. 02000000:00000010:3.0:1773941497.317586:0:14596:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 16384 at ffff988d18ba8000. 00000400:00000200:3.0:1773941497.317597:0:14596:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941497.317604:0:14596:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccdf20. 00000400:00000010:3.0:1773941497.317619:0:14596:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 184 at ffff988d02eb6840. 00000100:00000200:3.0:1773941497.317626:0:14596:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 16384 bytes, xid 1860112291473280, portal 25 00000400:00000010:3.0:1773941497.317635:0:14596:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac2a8. 00000100:00000200:3.0:1773941497.317639:0:14596:0:(niobuf.c:94:ptl_send_buf()) Sending 272 bytes to portal 26, xid 1860112291473280, offset 0 00000400:00000200:3.0:1773941497.317647:0:14596:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941497.317655:0:14596:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.317663:0:14596:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941497.317668:0:14596:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941497.317671:0:14596:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941497.317675:0:14596:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941497.317678:0:14596:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941497.317681:0:14596:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [85:85:256:4294967295] 192.168.204.114@tcp LPNI seq info [85:85:8:4294967295] 00000400:00000200:3.0:1773941497.317684:0:14596:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.317701:0:14596:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:3.0:1773941497.317709:0:14596:0:(socklnd_cb.c:1001:ksocknal_send()) sending 272 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:3.0:1773941497.317716:0:14596:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d21fada00. 00000800:00000200:3.0:1773941497.317750:0:14596:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:3.0:1773941497.317759:0:14596:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:3.0:1773941497.317766:0:14596:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d21fada00 type 1, nob 368 niov 1 nkiov 1 00000400:00000200:3.0:1773941497.317784:0:14596:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:1.0:1773941497.320392:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941497.320398:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d21fada00. 00000400:00000200:1.0:1773941497.320404:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941497.320411:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941497.320416:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac2a8 00000400:00000010:1.0:1773941497.320418:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac2a8. 00000100:00000200:1.0:1773941497.320427:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d111f1500 x1860112291473280/t0(0) o502->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941513 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'llog_process_th.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1773941497.321712:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.321722:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941497.321733:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.321740:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.321746:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.321754:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 8472 into portal 25 MB=0x69bc324602780 00000400:00000200:0.0:1773941497.321761:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.321768:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 8472/8472 into md 0x7b1 [4] + 224 00000400:00000200:0.0:1773941497.321793:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941497.321801:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:0.0:1773941497.321816:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d111f1500 x1860112291473280/t0(0) o502->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941513 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'llog_process_th.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1773941497.321881:0:14596:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccdf20. 00000400:00000200:3.0:1773941497.321887:0:14596:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d02eb6840 00000400:00000010:3.0:1773941497.321889:0:14596:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 184 at ffff988d02eb6840. 00000100:00000200:3.0:1773941497.321896:0:14596:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d111f1500 x1860112291473280/t0(0) o502->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941513 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'llog_process_th.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941497.321906:0:14596:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d111f1500 x1860112291473280/t0(0) o502->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8472 e 0 to 0 dl 1773941513 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'llog_process_th.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1773941497.321933:0:14596:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb69c0. 02000000:00000010:3.0:1773941497.321943:0:14596:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 16384 at ffff988d18ba8000. 02000000:00000010:3.0:1773941497.321955:0:14596:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065dda00. 00000100:00000010:3.0:1773941497.321991:0:14596:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d111f1500. 00000020:00000010:3.0:1773941497.322004:0:14596:0:(obd_config.c:2090:class_config_llog_handler()) kmalloced '(lcfg_new)': 200 at ffff988d21fad400. 00000020:00000010:3.0:1773941497.322008:0:14596:0:(obd_config.c:2123:class_config_llog_handler()) kfreed 'lcfg_new': 200 at ffff988d21fad400. 00000040:00000010:3.0:1773941497.322013:0:14596:0:(llog.c:891:llog_process_thread()) kvfree 'buf': 8192 at ffff988d20302000. 00000020:00000010:3.0:1773941497.322021:0:14596:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff988d02f4df80. 00000020:00000010:3.0:1773941497.322039:0:14596:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d1817de00. 00000040:00000010:3.0:1773941497.322044:0:14596:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d1817dc00. 00000020:00000010:3.0:1773941497.322049:0:14596:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d035c1800. 00000020:00000010:3.0:1773941497.322054:0:14596:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff988d03340f18. 00000020:00000010:3.0:1773941497.322060:0:14596:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d1817d800. 80000000:00000010:3.0:1773941497.322064:0:14596:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d18350720. 40000000:00000010:3.0:1773941497.322069:0:14596:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff988d02f4d480. 00100000:00000010:3.0:1773941497.322073:0:14596:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff988d035c1000. 00040000:00000010:3.0:1773941497.322077:0:14596:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d02cba400. 00040000:00000010:3.0:1773941497.322081:0:14596:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff988d035c0800. 00040000:00000010:3.0:1773941497.322085:0:14596:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d1817da00. 00000004:00000010:3.0:1773941497.322089:0:14596:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff988d035c0000. 00000004:00000010:3.0:1773941497.322094:0:14596:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff988d02d6a000. 00000004:00000010:3.0:1773941497.322100:0:14596:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff988d035c5000. 00002000:00000010:3.0:1773941497.322105:0:14596:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff988d054e0000. 00000004:00000010:3.0:1773941497.322109:0:14596:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff988d054e3800. 00000020:00000010:3.0:1773941497.322113:0:14596:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d1817d600. 00000040:00000010:2.0:1773941497.322159:0:14577:0:(llog.c:986:llog_process_or_fork()) kfreed 'lpi': 96 at ffff988d183505a0. 00000040:00000010:2.0:1773941497.322174:0:14577:0:(llog.c:70:llog_free_handle()) kvfree 'loghandle->lgh_hdr': 8192 at ffff988d20306000. 00000040:00000010:2.0:1773941497.322181:0:14577:0:(llog.c:72:llog_free_handle()) kfreed 'loghandle': 352 at ffff988d1817d400. 00000020:00000010:2.0:1773941497.322191:0:14577:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d1817ca00. 00000040:00000010:2.0:1773941497.322196:0:14577:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d1817ce00. 00000020:00000010:2.0:1773941497.322200:0:14577:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d035c2000. 00000020:00000010:2.0:1773941497.322206:0:14577:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d1817c600. 80000000:00000010:2.0:1773941497.322212:0:14577:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d18350960. 00040000:00000010:2.0:1773941497.322218:0:14577:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d1817cc00. 20000000:00000010:2.0:1773941497.322223:0:14577:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d1817d000. 00000020:00000010:2.0:1773941497.322226:0:14577:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d1817c200. 10000000:00000010:2.0:1773941497.322230:0:14577:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d02f4d200. 00000100:00000010:2.0:1773941497.322278:0:14577:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06505500. 02000000:00000010:2.0:1773941497.322295:0:14577:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d0627ae00. 00010000:00000010:2.0:1773941497.322309:0:14577:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d071eec00. 00010000:00000010:2.0:1773941497.322315:0:14577:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d089dfc00. 00000100:00000010:2.0:1773941497.322334:0:14577:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d188070c0. 02000000:00000010:2.0:1773941497.322356:0:14577:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a4400. 00000400:00000200:2.0:1773941497.322365:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941497.322369:0:14577:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380eec8. 00000400:00000010:2.0:1773941497.322375:0:14577:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000200:2.0:1773941497.322383:0:14577:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291473408, portal 25 00000400:00000010:2.0:1773941497.322389:0:14577:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516000. 00000100:00000200:2.0:1773941497.322392:0:14577:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291473408, offset 0 00000400:00000200:2.0:1773941497.322401:0:14577:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941497.322410:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941497.322421:0:14577:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941497.322429:0:14577:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941497.322441:0:14577:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941497.322449:0:14577:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941497.322455:0:14577:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:2.0:1773941497.322462:0:14577:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [86:86:256:4294967295] 192.168.204.114@tcp LPNI seq info [86:86:8:4294967295] 00000400:00000200:2.0:1773941497.322468:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941497.322483:0:14577:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:2.0:1773941497.322492:0:14577:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:2.0:1773941497.322499:0:14577:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d217a8d00. 00000800:00000200:2.0:1773941497.322511:0:14577:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:2.0:1773941497.322520:0:14577:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:2.0:1773941497.322527:0:14577:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d217a8d00 type 1, nob 424 niov 1 nkiov 1 00000400:00000200:2.0:1773941497.322554:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:1.0:1773941497.322616:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941497.322620:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d217a8d00. 00000400:00000200:1.0:1773941497.322624:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941497.322629:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941497.322632:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516000 00000400:00000010:1.0:1773941497.322635:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516000. 00000100:00000200:1.0:1773941497.322640:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06505500 x1860112291473408/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941513 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1773941497.323569:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.323576:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941497.323584:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.323596:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.323602:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.323608:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 344 into portal 25 MB=0x69bc324602800 00000400:00000200:0.0:1773941497.323612:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.323618:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 344/344 into md 0x7b9 [1] + 224 00000400:00000200:0.0:1773941497.323634:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941497.323640:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:0.0:1773941497.323651:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06505500 x1860112291473408/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941513 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1773941497.323709:0:14577:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380eec8. 00000400:00000200:3.0:1773941497.323714:0:14577:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:3.0:1773941497.323716:0:14577:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000200:3.0:1773941497.323722:0:14577:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06505500 x1860112291473408/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941513 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941497.323731:0:14577:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06505500 x1860112291473408/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941513 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1773941497.323752:0:14577:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d188070c0. 02000000:00000010:3.0:1773941497.323773:0:14577:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a4400. 02000000:00000010:3.0:1773941497.323776:0:14577:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d0627ae00. 00000100:00000010:3.0:1773941497.323779:0:14577:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06505500. 10000000:00000010:3.0:1773941497.323811:0:14577:0:(mgc_request.c:1545:mgc_process_recover_log()) kmalloced '(pages)': 2048 at ffff988d035c2000. 00000100:00000010:3.0:1773941497.324437:0:14577:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d111f1500. 02000000:00000010:3.0:1773941497.324449:0:14577:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065dda00. 00000100:00000010:3.0:1773941497.324458:0:14577:0:(client.c:169:ptlrpc_new_bulk()) kmalloced '(desc)': 984 at ffff988d111d2000. 00000100:00000010:3.0:1773941497.324463:0:14577:0:(client.c:174:ptlrpc_new_bulk()) kmalloced '(desc->bd_vec)': 4096 at ffff988d02ecd000. 00000100:00000010:3.0:1773941497.324475:0:14577:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb69c0. 00000400:00000200:3.0:1773941497.324489:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941497.324496:0:14577:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccdf20. 00000400:00000010:3.0:1773941497.324504:0:14577:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 4216 at ffff988d0646c000. 00000100:00000200:3.0:1773941497.324510:0:14577:0:(niobuf.c:453:ptlrpc_register_bulk()) Setup 1 bulk put-sink buffers: 256 pages 1048576 bytes, mbits x0x69bc324602880-0x69bc324602880, portal 33 02000000:00000010:3.0:1773941497.324516:0:14577:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff988d065dce00. 00000400:00000200:3.0:1773941497.324519:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941497.324521:0:14577:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd8f0. 00000400:00000010:3.0:1773941497.324524:0:14577:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ace58. 00000100:00000200:3.0:1773941497.324526:0:14577:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112291473536, portal 25 00000400:00000010:3.0:1773941497.324530:0:14577:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acb28. 00000100:00000200:3.0:1773941497.324533:0:14577:0:(niobuf.c:94:ptl_send_buf()) Sending 304 bytes to portal 26, xid 1860112291473536, offset 0 00000400:00000200:3.0:1773941497.324538:0:14577:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941497.324542:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.324546:0:14577:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941497.324549:0:14577:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941497.324552:0:14577:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941497.324558:0:14577:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941497.324561:0:14577:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941497.324567:0:14577:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [87:87:256:4294967295] 192.168.204.114@tcp LPNI seq info [87:87:8:4294967295] 00000400:00000200:3.0:1773941497.324571:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.324580:0:14577:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:3.0:1773941497.324586:0:14577:0:(socklnd_cb.c:1001:ksocknal_send()) sending 304 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:3.0:1773941497.324590:0:14577:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d21fad400. 00000800:00000200:3.0:1773941497.324597:0:14577:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:3.0:1773941497.324603:0:14577:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:3.0:1773941497.324612:0:14577:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d21fad400 type 1, nob 400 niov 1 nkiov 1 00000400:00000200:3.0:1773941497.324626:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:1.0:1773941497.324719:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941497.324723:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d21fad400. 00000400:00000200:1.0:1773941497.324727:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941497.324734:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941497.324737:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acb28 00000400:00000010:1.0:1773941497.324739:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acb28. 00000100:00000200:1.0:1773941497.324745:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d111f1500 x1860112291473536/t0(0) o256->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 304/240 e 0 to 0 dl 1773941513 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1773941497.325693:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.325698:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941497.325702:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.325704:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.325706:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.325708:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 132 into portal 33 MB=0x69bc324602880 00000400:00000200:0.0:1773941497.325711:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.325714:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 21 from 12345-192.168.204.114@tcp of length 132/132 into md 0x7c1 [256] + 0 00000400:00000010:0.0:1773941497.325717:0:2393:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccdf20. 00000400:00000200:0.0:1773941497.325721:0:2393:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md ffff988d0646c000 00000400:00000200:0.0:1773941497.325726:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941497.325729:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941497.325733:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d0646c000 00000400:00000010:0.0:1773941497.325734:0:2393:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 4216 at ffff988d0646c000. 00000100:00000200:0.0:1773941497.325739:0:2393:0:(events.c:192:client_bulk_callback()) event type 2, status 0, req ffff988d111f1500 desc ffff988d111d2000 mbits 1860112291473536 00000400:00000200:0.0:1773941497.325751:0:2393:0:(lib-md.c:96:lnet_cpt_of_md()) Couldn't resolve first page of md 0000000000000000 with offset 0 00000400:00000200:0.0:1773941497.325754:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.325757:0:2393:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source Specified: 192.168.204.14@tcp to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941497.325762:0:2393:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [88:88:256:4294967295] 192.168.204.114@tcp LPNI seq info [88:88:8:4294967295] 00000400:00000200:0.0:1773941497.325765:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.325770:0:2393:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:192.168.204.14@tcp) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) 192.168.204.114@tcp : ACK try# 0 00000800:00000200:0.0:1773941497.325781:0:2393:0:(socklnd_cb.c:1001:ksocknal_send()) sending 0 bytes in 256 frags to 12345-192.168.204.114@tcp 00000800:00000010:0.0:1773941497.325786:0:2393:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 4328 at ffff988d10804000. 00000800:00000200:0.0:1773941497.325790:0:2393:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:0.0:1773941497.325793:0:2393:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:0.0:1773941497.325798:0:2393:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d10804000 type 0, nob 96 niov 1 nkiov 0 00000800:00000200:0.0:1773941497.325851:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941497.325854:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 4328 at ffff988d10804000. 00000400:00000200:0.0:1773941497.325862:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941497.325867:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: ACK: OK 00000400:00000200:0.0:1773941497.326497:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.326503:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941497.326510:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.326514:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.326521:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.326528:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 240 into portal 25 MB=0x69bc324602880 00000400:00000200:0.0:1773941497.326533:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.326539:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 240/240 into md 0x7c9 [1] + 224 00000400:00000200:0.0:1773941497.326550:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941497.326555:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:0.0:1773941497.326567:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d111f1500 x1860112291473536/t0(0) o256->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 304/240 e 0 to 0 dl 1773941513 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1773941497.326617:0:14577:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd8f0. 00000400:00000200:3.0:1773941497.326623:0:14577:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ace58 00000400:00000010:3.0:1773941497.326625:0:14577:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ace58. 00000100:00000200:3.0:1773941497.326636:0:14577:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d111f1500 x1860112291473536/t0(0) o256->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 304/240 e 0 to 0 dl 1773941513 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941497.326647:0:14577:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d111f1500 x1860112291473536/t0(0) o256->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 304/240 e 0 to 0 dl 1773941513 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1773941497.326675:0:14577:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb69c0. 10000000:00000010:3.0:1773941497.326687:0:14577:0:(mgc_request.c:1297:mgc_apply_recover_logs()) kmalloced '(buf)': 4096 at ffff988d02ecc000. 00000100:00000200:3.0:1773941497.326699:0:14577:0:(lustre_peer.c:175:class_add_nids_to_uuid()) Comparing 192.168.204.114@tcp with 192.168.204.114@tcp 00000100:00000200:3.0:1773941497.326702:0:14577:0:(lustre_peer.c:182:class_add_nids_to_uuid()) Updating UUID '192.168.204.114@tcp' 00000400:00000200:3.0:1773941497.326707:0:14577:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:3.0:1773941497.326711:0:14577:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.204.114@tcp 00000400:00000200:3.0:1773941497.326716:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.326722:0:14577:0:(peer.c:1737:lnet_peer_add()) peer 192.168.204.114@tcp NID flags 0x100001: -17 00000400:00000200:3.0:1773941497.326727:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.326736:0:14577:0:(peer.c:2767:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1773941497.326739:0:14577:0:(peer.c:2776:lnet_discover_peer_locked()) peer 192.168.204.114@tcp NID 192.168.204.114@tcp: 0. pending discovery 00000400:00000200:3.0:1773941497.326742:0:14577:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.204.114@tcp 10000000:00000010:3.0:1773941497.326748:0:14577:0:(mgc_request.c:1479:mgc_apply_recover_logs()) kmalloced '(lcfg)': 128 at ffff988d03580580. 00000020:00000010:3.0:1773941497.326757:0:14577:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d065dd400. 00000020:00000010:3.0:1773941497.326761:0:14577:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d065ddc00. 00000020:00000010:3.0:1773941497.326766:0:14577:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d20936240. 00000008:00000010:3.0:1773941497.326774:0:14577:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff988d113fd940. 00020000:00000010:3.0:1773941497.326779:0:14577:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff988d20218840. 00008000:00000010:3.0:1773941497.326785:0:14577:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d107f2de8. 00000020:00000010:3.0:1773941497.326843:0:14577:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d065ddc00. 00000020:00000010:3.0:1773941497.326850:0:14577:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d20936240. 00000008:00000010:3.0:1773941497.326855:0:14577:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff988d113fd940. 00020000:00000010:3.0:1773941497.326861:0:14577:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff988d20218840. 00008000:00000010:3.0:1773941497.326877:0:14577:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d107f2de8. 00000020:00000010:3.0:1773941497.326884:0:14577:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d065dd400. 10000000:00000010:3.0:1773941497.326892:0:14577:0:(mgc_request.c:1495:mgc_apply_recover_logs()) kfreed 'lcfg': 128 at ffff988d03580580. 00000100:00000200:3.0:1773941497.326903:0:14577:0:(lustre_peer.c:175:class_add_nids_to_uuid()) Comparing 192.168.204.114@tcp with 192.168.204.114@tcp 00000100:00000200:3.0:1773941497.326906:0:14577:0:(lustre_peer.c:182:class_add_nids_to_uuid()) Updating UUID '192.168.204.114@tcp' 00000400:00000200:3.0:1773941497.326908:0:14577:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:3.0:1773941497.326912:0:14577:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.204.114@tcp 00000400:00000200:3.0:1773941497.326918:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.326921:0:14577:0:(peer.c:1737:lnet_peer_add()) peer 192.168.204.114@tcp NID flags 0x100001: -17 00000400:00000200:3.0:1773941497.326927:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.326936:0:14577:0:(peer.c:2767:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1773941497.326939:0:14577:0:(peer.c:2776:lnet_discover_peer_locked()) peer 192.168.204.114@tcp NID 192.168.204.114@tcp: 0. pending discovery 00000400:00000200:3.0:1773941497.326942:0:14577:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.204.114@tcp 10000000:00000010:3.0:1773941497.326951:0:14577:0:(mgc_request.c:1479:mgc_apply_recover_logs()) kmalloced '(lcfg)': 128 at ffff988d03580580. 00000020:00000010:3.0:1773941497.326956:0:14577:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d065dd400. 00000020:00000010:3.0:1773941497.326979:0:14577:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d065ddc00. 00000020:00000010:3.0:1773941497.326983:0:14577:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d20936240. 00000008:00000010:3.0:1773941497.326987:0:14577:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff988d113fd940. 00020000:00000010:3.0:1773941497.326991:0:14577:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff988d20218840. 00008000:00000010:3.0:1773941497.326994:0:14577:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d107f2de8. 00000020:00000010:3.0:1773941497.327025:0:14577:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d065ddc00. 00000020:00000010:3.0:1773941497.327028:0:14577:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d20936240. 00000008:00000010:3.0:1773941497.327033:0:14577:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff988d113fd940. 00020000:00000010:3.0:1773941497.327038:0:14577:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff988d20218840. 00008000:00000010:3.0:1773941497.327041:0:14577:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d107f2de8. 00000020:00000010:3.0:1773941497.327044:0:14577:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d065dd400. 10000000:00000010:3.0:1773941497.327047:0:14577:0:(mgc_request.c:1495:mgc_apply_recover_logs()) kfreed 'lcfg': 128 at ffff988d03580580. 00000100:00000200:3.0:1773941497.327053:0:14577:0:(lustre_peer.c:175:class_add_nids_to_uuid()) Comparing 192.168.204.114@tcp with 192.168.204.114@tcp 00000100:00000200:3.0:1773941497.327055:0:14577:0:(lustre_peer.c:182:class_add_nids_to_uuid()) Updating UUID '192.168.204.114@tcp' 00000400:00000200:3.0:1773941497.327057:0:14577:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00000200:3.0:1773941497.327060:0:14577:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.204.114@tcp 00000400:00000200:3.0:1773941497.327064:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.327068:0:14577:0:(peer.c:1737:lnet_peer_add()) peer 192.168.204.114@tcp NID flags 0x100001: -17 00000400:00000200:3.0:1773941497.327075:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.327089:0:14577:0:(peer.c:2767:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:3.0:1773941497.327092:0:14577:0:(peer.c:2776:lnet_discover_peer_locked()) peer 192.168.204.114@tcp NID 192.168.204.114@tcp: 0. pending discovery 00000400:00000200:3.0:1773941497.327096:0:14577:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.204.114@tcp 10000000:00000010:3.0:1773941497.327100:0:14577:0:(mgc_request.c:1479:mgc_apply_recover_logs()) kmalloced '(lcfg)': 128 at ffff988d03580580. 00000020:00000010:3.0:1773941497.327105:0:14577:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d065dd400. 00000020:00000010:3.0:1773941497.327108:0:14577:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d065ddc00. 00000020:00000010:3.0:1773941497.327110:0:14577:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d20936240. 00000008:00000010:3.0:1773941497.327114:0:14577:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff988d113fd940. 00020000:00000010:3.0:1773941497.327116:0:14577:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff988d20218840. 00008000:00000010:3.0:1773941497.327119:0:14577:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d107f2de8. 00000020:00000010:3.0:1773941497.327152:0:14577:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d065ddc00. 00000020:00000010:3.0:1773941497.327155:0:14577:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d20936240. 00000008:00000010:3.0:1773941497.327158:0:14577:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff988d113fd940. 00020000:00000010:3.0:1773941497.327162:0:14577:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff988d20218840. 00008000:00000010:3.0:1773941497.327167:0:14577:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d107f2de8. 00000020:00000010:3.0:1773941497.327173:0:14577:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d065dd400. 10000000:00000010:3.0:1773941497.327177:0:14577:0:(mgc_request.c:1495:mgc_apply_recover_logs()) kfreed 'lcfg': 128 at ffff988d03580580. 10000000:00000010:3.0:1773941497.327181:0:14577:0:(mgc_request.c:1507:mgc_apply_recover_logs()) kfreed 'buf': 4096 at ffff988d02ecc000. 02000000:00000010:3.0:1773941497.327186:0:14577:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff988d065dce00. 00000100:00000010:3.0:1773941497.327200:0:14577:0:(client.c:333:ptlrpc_free_bulk()) kvfree 'desc->bd_vec': 4096 at ffff988d02ecd000. 00000100:00000010:3.0:1773941497.327207:0:14577:0:(client.c:334:ptlrpc_free_bulk()) kfreed 'desc': 984 at ffff988d111d2000. 02000000:00000010:3.0:1773941497.327211:0:14577:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065dda00. 00000100:00000010:3.0:1773941497.327214:0:14577:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d111f1500. 10000000:00000010:3.0:1773941497.327857:0:14577:0:(mgc_request.c:1663:mgc_process_recover_log()) kvfree 'pages': 2048 at ffff988d035c2000. 00000100:00000010:3.0:1773941497.327880:0:14577:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d111f1500. 02000000:00000010:3.0:1773941497.327887:0:14577:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065dda00. 00010000:00000010:3.0:1773941497.327899:0:14577:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d1a01da00. 00010000:00000010:3.0:1773941497.327904:0:14577:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d0168b400. 00000100:00000010:3.0:1773941497.327917:0:14577:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb69c0. 02000000:00000010:3.0:1773941497.327926:0:14577:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d111d2000. 00000400:00000200:3.0:1773941497.327930:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941497.327933:0:14577:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd8f0. 00000400:00000010:3.0:1773941497.327938:0:14577:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ace58. 00000100:00000200:3.0:1773941497.327950:0:14577:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291473664, portal 25 00000400:00000010:3.0:1773941497.327955:0:14577:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acaa0. 00000100:00000200:3.0:1773941497.327959:0:14577:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291473664, offset 0 00000400:00000200:3.0:1773941497.327997:0:14577:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941497.328004:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.328012:0:14577:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941497.328018:0:14577:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941497.328023:0:14577:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941497.328027:0:14577:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941497.328032:0:14577:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941497.328036:0:14577:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [89:89:256:4294967295] 192.168.204.114@tcp LPNI seq info [89:89:8:4294967295] 00000400:00000200:3.0:1773941497.328040:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.328046:0:14577:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:3.0:1773941497.328050:0:14577:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:3.0:1773941497.328054:0:14577:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d21fad000. 00000800:00000200:3.0:1773941497.328058:0:14577:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:3.0:1773941497.328062:0:14577:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:3.0:1773941497.328064:0:14577:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d21fad000 type 1, nob 424 niov 1 nkiov 1 00000400:00000200:3.0:1773941497.328077:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:1.0:1773941497.328142:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941497.328147:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d21fad000. 00000400:00000200:1.0:1773941497.328151:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941497.328158:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941497.328162:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acaa0 00000400:00000010:1.0:1773941497.328165:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acaa0. 00000100:00000200:1.0:1773941497.328173:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d111f1500 x1860112291473664/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941513 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1773941497.329158:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941497.329168:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941497.329174:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941497.329177:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941497.329181:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941497.329184:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 344 into portal 25 MB=0x69bc324602900 00000400:00000200:1.0:1773941497.329194:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941497.329198:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 344/344 into md 0x7d1 [1] + 224 00000400:00000200:1.0:1773941497.329212:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941497.329215:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:1.0:1773941497.329220:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d111f1500 x1860112291473664/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941513 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1773941497.329299:0:14577:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd8f0. 00000400:00000200:2.0:1773941497.329307:0:14577:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ace58 00000400:00000010:2.0:1773941497.329310:0:14577:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ace58. 00000100:00000200:2.0:1773941497.329319:0:14577:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d111f1500 x1860112291473664/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941513 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941497.329332:0:14577:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d111f1500 x1860112291473664/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941513 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1773941497.329360:0:14577:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb69c0. 02000000:00000010:2.0:1773941497.329384:0:14577:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d111d2000. 02000000:00000010:2.0:1773941497.329392:0:14577:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065dda00. 00000100:00000010:2.0:1773941497.329397:0:14577:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d111f1500. 10000000:00000010:2.0:1773941497.329408:0:14577:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d05437940. 00000020:00000010:2.0:1773941497.329413:0:14577:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d0627b000. 00000020:00000010:2.0:1773941497.329419:0:14577:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d0627ae00. 00000040:00000010:2.0:1773941497.329424:0:14577:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d0627a800. 00000020:00000010:2.0:1773941497.329434:0:14577:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d0544a000. 00000020:00000010:2.0:1773941497.329442:0:14577:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d1a004200. 80000000:00000010:2.0:1773941497.329447:0:14577:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d08bbb7e0. 00040000:00000010:2.0:1773941497.329463:0:14577:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d02cbde00. 20000000:00000010:2.0:1773941497.329469:0:14577:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d02cbc200. 00000040:00000010:2.0:1773941497.329478:0:14577:0:(llog.c:42:llog_alloc_handle()) kmalloced '(loghandle)': 352 at ffff988d02cbce00. 00000100:00000010:2.0:1773941497.329486:0:14577:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06504700. 02000000:00000010:2.0:1773941497.329498:0:14577:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d02cbda00. 00000100:00000010:2.0:1773941497.329509:0:14577:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d18807900. 02000000:00000010:2.0:1773941497.329524:0:14577:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff988d1a8abc00. 00000400:00000200:2.0:1773941497.329530:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941497.329535:0:14577:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e2c0. 00000400:00000010:2.0:1773941497.329545:0:14577:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516908. 00000100:00000200:2.0:1773941497.329551:0:14577:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112291473792, portal 25 00000400:00000010:2.0:1773941497.329557:0:14577:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516dd0. 00000100:00000200:2.0:1773941497.329560:0:14577:0:(niobuf.c:94:ptl_send_buf()) Sending 504 bytes to portal 26, xid 1860112291473792, offset 0 00000400:00000200:2.0:1773941497.329570:0:14577:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941497.329577:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941497.329585:0:14577:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941497.329589:0:14577:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941497.329594:0:14577:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941497.329597:0:14577:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941497.329603:0:14577:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:2.0:1773941497.329608:0:14577:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [90:90:256:4294967295] 192.168.204.114@tcp LPNI seq info [90:90:8:4294967295] 00000400:00000200:2.0:1773941497.329613:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941497.329629:0:14577:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:2.0:1773941497.329636:0:14577:0:(socklnd_cb.c:1001:ksocknal_send()) sending 504 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:2.0:1773941497.329642:0:14577:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d217a8b00. 00000800:00000200:2.0:1773941497.329651:0:14577:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:2.0:1773941497.329658:0:14577:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:2.0:1773941497.329663:0:14577:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d217a8b00 type 1, nob 600 niov 1 nkiov 1 00000400:00000200:2.0:1773941497.329681:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:1.0:1773941497.329779:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941497.329783:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d217a8b00. 00000400:00000200:1.0:1773941497.329788:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941497.329793:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941497.329795:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516dd0 00000400:00000010:1.0:1773941497.329799:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516dd0. 00000100:00000200:1.0:1773941497.329805:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06504700 x1860112291473792/t0(0) o501->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 504/272 e 0 to 0 dl 1773941513 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1773941497.330609:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941497.330614:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941497.330619:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941497.330622:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941497.330625:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941497.330628:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 272 into portal 25 MB=0x69bc324602980 00000400:00000200:1.0:1773941497.330630:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941497.330633:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 272/272 into md 0x7d9 [1] + 224 00000400:00000200:1.0:1773941497.330643:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941497.330646:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:1.0:1773941497.330651:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06504700 x1860112291473792/t0(0) o501->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 504/272 e 0 to 0 dl 1773941513 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1773941497.330708:0:14577:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e2c0. 00000400:00000200:2.0:1773941497.330714:0:14577:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516908 00000400:00000010:2.0:1773941497.330724:0:14577:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516908. 00000100:00000200:2.0:1773941497.330732:0:14577:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06504700 x1860112291473792/t0(0) o501->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 504/272 e 0 to 0 dl 1773941513 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941497.330742:0:14577:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06504700 x1860112291473792/t0(0) o501->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 504/272 e 0 to 0 dl 1773941513 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000010:2.0:1773941497.330762:0:14577:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d18807900. 02000000:00000010:2.0:1773941497.330769:0:14577:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff988d1a8abc00. 02000000:00000010:2.0:1773941497.330775:0:14577:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d02cbda00. 00000100:00000010:2.0:1773941497.330778:0:14577:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06504700. 00000040:00000010:2.0:1773941497.330788:0:14577:0:(llog.c:412:llog_init_handle()) kmalloced '(llh)': 8192 at ffff988d18b16000. 00000100:00000010:2.0:1773941497.330797:0:14577:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06504700. 02000000:00000010:2.0:1773941497.330806:0:14577:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d1a8abc00. 00000100:00000010:2.0:1773941497.330814:0:14577:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d18807900. 02000000:00000010:2.0:1773941497.330840:0:14577:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 16384 at ffff988d18bc0000. 00000400:00000200:2.0:1773941497.330847:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941497.330851:0:14577:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e2c0. 00000400:00000010:2.0:1773941497.330859:0:14577:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 184 at ffff988d188070c0. 00000100:00000200:2.0:1773941497.330863:0:14577:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 16384 bytes, xid 1860112291473920, portal 25 00000400:00000010:2.0:1773941497.330867:0:14577:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516908. 00000100:00000200:2.0:1773941497.330871:0:14577:0:(niobuf.c:94:ptl_send_buf()) Sending 272 bytes to portal 26, xid 1860112291473920, offset 0 00000400:00000200:2.0:1773941497.330878:0:14577:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941497.330882:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941497.330887:0:14577:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941497.330891:0:14577:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941497.330895:0:14577:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941497.330899:0:14577:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941497.330905:0:14577:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:2.0:1773941497.330909:0:14577:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [91:91:256:4294967295] 192.168.204.114@tcp LPNI seq info [91:91:8:4294967295] 00000400:00000200:2.0:1773941497.330916:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941497.330927:0:14577:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:2.0:1773941497.330934:0:14577:0:(socklnd_cb.c:1001:ksocknal_send()) sending 272 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:2.0:1773941497.330950:0:14577:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d217a8a00. 00000800:00000200:2.0:1773941497.330956:0:14577:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:2.0:1773941497.330993:0:14577:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:2.0:1773941497.330996:0:14577:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d217a8a00 type 1, nob 368 niov 1 nkiov 1 00000400:00000200:2.0:1773941497.331019:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:1.0:1773941497.331055:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941497.331058:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d217a8a00. 00000400:00000200:1.0:1773941497.331063:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941497.331067:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941497.331075:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516908 00000400:00000010:1.0:1773941497.331077:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516908. 00000100:00000200:1.0:1773941497.331085:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06504700 x1860112291473920/t0(0) o503->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8416 e 0 to 0 dl 1773941513 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1773941497.332061:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941497.332067:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941497.332075:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941497.332081:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941497.332087:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941497.332092:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 8416 into portal 25 MB=0x69bc324602a00 00000400:00000200:1.0:1773941497.332097:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941497.332101:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 8416/8416 into md 0x7e1 [4] + 224 00000400:00000200:1.0:1773941497.332118:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941497.332123:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:1.0:1773941497.332133:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06504700 x1860112291473920/t0(0) o503->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8416 e 0 to 0 dl 1773941513 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1773941497.332180:0:14577:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e2c0. 00000400:00000200:3.0:1773941497.332189:0:14577:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d188070c0 00000400:00000010:3.0:1773941497.332191:0:14577:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 184 at ffff988d188070c0. 00000100:00000200:3.0:1773941497.332204:0:14577:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06504700 x1860112291473920/t0(0) o503->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8416 e 0 to 0 dl 1773941513 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941497.332219:0:14577:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06504700 x1860112291473920/t0(0) o503->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 272/8416 e 0 to 0 dl 1773941513 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1773941497.332240:0:14577:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d18807900. 02000000:00000010:3.0:1773941497.332265:0:14577:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 16384 at ffff988d18bc0000. 02000000:00000010:3.0:1773941497.332278:0:14577:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d1a8abc00. 00000100:00000010:3.0:1773941497.332282:0:14577:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06504700. 00000040:00000010:3.0:1773941497.332297:0:14577:0:(llog.c:945:llog_process_or_fork()) kmalloced '(lpi)': 96 at ffff988d20936240. 00000020:00000010:3.0:1773941497.332447:0:14602:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d1817c200. 00000020:00000010:3.0:1773941497.332454:0:14602:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d1817d000. 00000040:00000010:3.0:1773941497.332460:0:14602:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d1817cc00. 00000020:00000010:3.0:1773941497.332466:0:14602:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d035c2000. 00000020:00000010:3.0:1773941497.332473:0:14602:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff988d03340f18. 00000020:00000010:3.0:1773941497.332480:0:14602:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d1817c600. 80000000:00000010:3.0:1773941497.332486:0:14602:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d209362a0. 40000000:00000010:3.0:1773941497.332492:0:14602:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff988d02f4d200. 00100000:00000010:3.0:1773941497.332499:0:14602:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff988d035c5000. 00040000:00000010:3.0:1773941497.332506:0:14602:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d21fad000. 00040000:00000010:3.0:1773941497.332513:0:14602:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff988d035c0000. 00040000:00000010:3.0:1773941497.332519:0:14602:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d1817ce00. 00000004:00000010:3.0:1773941497.332525:0:14602:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff988d035c0800. 00000004:00000010:3.0:1773941497.332532:0:14602:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff988d02d6a000. 00000004:00000010:3.0:1773941497.332537:0:14602:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff988d035c1000. 00002000:00000010:3.0:1773941497.332541:0:14602:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff988d111d3000. 00000004:00000010:3.0:1773941497.332547:0:14602:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff988d111d2000. 00000020:00000010:3.0:1773941497.332552:0:14602:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff988d02f4d480. 00000040:00000010:3.0:1773941497.332561:0:14602:0:(llog.c:570:llog_process_thread()) kmalloced '(buf)': 8192 at ffff988d06468000. 00000040:00000010:3.0:1773941497.332958:0:14602:0:(llog.c:891:llog_process_thread()) kvfree 'buf': 8192 at ffff988d06468000. 00000020:00000010:3.0:1773941497.332995:0:14602:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff988d02f4d480. 00000020:00000010:3.0:1773941497.333002:0:14602:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d1817d000. 00000040:00000010:3.0:1773941497.333005:0:14602:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d1817cc00. 00000020:00000010:3.0:1773941497.333008:0:14602:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d035c2000. 00000020:00000010:3.0:1773941497.333011:0:14602:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff988d03340f18. 00000020:00000010:3.0:1773941497.333016:0:14602:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d1817c600. 80000000:00000010:3.0:1773941497.333018:0:14602:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d209362a0. 40000000:00000010:3.0:1773941497.333022:0:14602:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff988d02f4d200. 00100000:00000010:3.0:1773941497.333025:0:14602:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff988d035c5000. 00040000:00000010:3.0:1773941497.333028:0:14602:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d21fad000. 00040000:00000010:3.0:1773941497.333033:0:14602:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff988d035c0000. 00040000:00000010:3.0:1773941497.333036:0:14602:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d1817ce00. 00000004:00000010:3.0:1773941497.333039:0:14602:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff988d035c0800. 00000004:00000010:3.0:1773941497.333048:0:14602:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff988d02d6a000. 00000004:00000010:3.0:1773941497.333054:0:14602:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff988d035c1000. 00002000:00000010:3.0:1773941497.333058:0:14602:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff988d111d3000. 00000004:00000010:3.0:1773941497.333061:0:14602:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff988d111d2000. 00000020:00000010:3.0:1773941497.333066:0:14602:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d1817c200. 00000040:00000010:3.0:1773941497.333081:0:14577:0:(llog.c:986:llog_process_or_fork()) kfreed 'lpi': 96 at ffff988d20936240. 00000040:00000010:3.0:1773941497.333086:0:14577:0:(llog.c:70:llog_free_handle()) kvfree 'loghandle->lgh_hdr': 8192 at ffff988d18b16000. 00000040:00000010:3.0:1773941497.333090:0:14577:0:(llog.c:72:llog_free_handle()) kfreed 'loghandle': 352 at ffff988d02cbce00. 00000020:00000010:3.0:1773941497.333097:0:14577:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d0627ae00. 00000040:00000010:3.0:1773941497.333100:0:14577:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d0627a800. 00000020:00000010:3.0:1773941497.333104:0:14577:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d0544a000. 00000020:00000010:3.0:1773941497.333109:0:14577:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d1a004200. 80000000:00000010:3.0:1773941497.333114:0:14577:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d08bbb7e0. 00040000:00000010:3.0:1773941497.333117:0:14577:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d02cbde00. 20000000:00000010:3.0:1773941497.333121:0:14577:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d02cbc200. 00000020:00000010:3.0:1773941497.333125:0:14577:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0627b000. 10000000:00000010:3.0:1773941497.333128:0:14577:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d05437940. 00000020:00000010:3.0:1773941497.333148:0:14577:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d065c5000. 00000040:00000010:3.0:1773941497.333151:0:14577:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d065c5800. 00000020:00000010:3.0:1773941497.333155:0:14577:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d05448800. 00000020:00000010:3.0:1773941497.333159:0:14577:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d065c4e00. 00040000:00000010:3.0:1773941497.333163:0:14577:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d0a0db600. 00040000:00000010:3.0:1773941497.333168:0:14577:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d065c4c00. 00000020:00000010:3.0:1773941497.333171:0:14577:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d11266a00. 00000020:00000010:3.0:1773941497.333176:0:14577:0:(obd_mount.c:82:lustre_process_log()) kfreed 'lcfg': 184 at ffff988d18807300. 00000020:00000010:3.0:1773941497.333180:0:14577:0:(obd_mount.c:84:lustre_process_log()) kfreed 'bufs': 104 at ffff988d107f4e00. 00000080:00000010:3.0:1773941497.333189:0:14577:0:(llite_lib.c:1502:ll_fill_super()) kmalloced '(dt)': 33 at ffff988d02f4d200. 00000080:00000010:3.0:1773941497.333193:0:14577:0:(llite_lib.c:1508:ll_fill_super()) kmalloced '(md)': 33 at ffff988d082b2f80. 00000080:00000010:3.0:1773941497.333200:0:14577:0:(llite_lib.c:348:client_common_fill_super()) kmalloced '(data)': 192 at ffff988d02eb6840. 00000080:00000010:3.0:1773941497.333203:0:14577:0:(llite_lib.c:352:client_common_fill_super()) kmalloced '(osfs)': 144 at ffff988d02eb69c0. 00000020:00000010:3.0:1773941497.333211:0:14577:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff988d035c1000. 00000020:00000010:3.0:1773941497.333259:0:14577:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff988d035c0800. 00000400:00000200:3.0:1773941497.333275:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.333281:0:14577:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000010:3.0:1773941497.333309:0:14577:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d111f2680. 02000000:00000010:3.0:1773941497.333320:0:14577:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d111d2000. 40000000:00000010:3.0:1773941497.333368:0:14577:0:(fid_request.c:432:client_fid_init()) kmalloced '(cli->cl_seq)': 224 at ffff988d21fad000. 40000000:00000010:3.0:1773941497.333373:0:14577:0:(fid_request.c:436:client_fid_init()) kmalloced '(prefix)': 133 at ffff988d1837d0c0. 40000000:00000010:3.0:1773941497.333446:0:14577:0:(fid_request.c:444:client_fid_init()) kfreed 'prefix': 133 at ffff988d1837d0c0. 80000000:00000010:3.0:1773941497.333453:0:14577:0:(fld_request.c:150:fld_client_add_target()) kmalloced '(target)': 40 at ffff988d082b20c0. 00020000:00000010:3.0:1773941497.333462:0:14577:0:(lu_tgt_descs.c:102:lu_qos_add_tgt()) kmalloced '(svr)': 104 at ffff988d03580580. 00000100:00000010:3.0:1773941497.333470:0:14577:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d111f3100. 02000000:00000010:3.0:1773941497.333476:0:14577:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff988d21fad400. 00000020:00000010:3.0:1773941497.333509:0:14577:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff988d035c0000. 00000008:00000010:2.0:1773941497.333517:0:2403:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff988d18280000. 00000400:00000200:3.0:1773941497.333524:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00020000:00000010:2.0:1773941497.333527:0:2403:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff988d035a1000. 00000400:00000200:3.0:1773941497.333528:0:14577:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000010:3.0:1773941497.333538:0:14577:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d111f3800. 00000008:00000010:2.0:1773941497.333541:0:2403:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff988d03545b80. 02000000:00000010:3.0:1773941497.333545:0:14577:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d111d3000. 00020000:00000010:2.0:1773941497.333551:0:2403:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff988d02ca1f80. 40000000:00000010:3.0:1773941497.333565:0:14577:0:(fid_request.c:432:client_fid_init()) kmalloced '(cli->cl_seq)': 224 at ffff988d21fada00. 40000000:00000010:3.0:1773941497.333569:0:14577:0:(fid_request.c:436:client_fid_init()) kmalloced '(prefix)': 133 at ffff988d1837d0c0. 40000000:00000010:3.0:1773941497.333647:0:14577:0:(fid_request.c:444:client_fid_init()) kfreed 'prefix': 133 at ffff988d1837d0c0. 80000000:00000010:3.0:1773941497.333651:0:14577:0:(fld_request.c:150:fld_client_add_target()) kmalloced '(target)': 40 at ffff988d03ec7540. 00000100:00000010:3.0:1773941497.333656:0:14577:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d111f0380. 02000000:00000010:3.0:1773941497.333662:0:14577:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff988d0a0db600. 00800000:00000010:3.0:1773941497.333688:0:14577:0:(lmv_obd.c:1417:lmv_statfs()) kmalloced '(temp)': 144 at ffff988d1837d0c0. 00000100:00000010:3.0:1773941497.333693:0:14577:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d111f1c00. 02000000:00000010:3.0:1773941497.333701:0:14577:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff988d02cba400. 00000100:00000010:3.0:1773941497.333707:0:14577:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1837d000. 00000008:00000010:3.0:1773941497.333731:0:2400:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff988d113fd940. 00020000:00000010:3.0:1773941497.333735:0:2400:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff988d20218840. 00000008:00000010:3.0:1773941497.333739:0:2400:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff988d035cadc0. 00020000:00000010:3.0:1773941497.333743:0:2400:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff988d113f5d40. 02000000:00000010:3.0:1773941497.333752:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d111d1c00. 00000400:00000200:3.0:1773941497.333759:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941497.333762:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccdd68. 00000400:00000010:3.0:1773941497.333766:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac220. 00000100:00000200:3.0:1773941497.333777:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291474048, portal 10 00000020:00000010:3.0:1773941497.333783:0:2400:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffff988d06468000. 00000400:00000010:3.0:1773941497.333788:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acd48. 00000100:00000200:3.0:1773941497.333791:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1860112291474048, offset 0 00000400:00000200:3.0:1773941497.333797:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941497.333802:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.333807:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941497.333812:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941497.333816:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941497.333821:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941497.333825:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941497.333830:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [92:92:256:4294967295] 192.168.204.114@tcp LPNI seq info [92:92:8:4294967295] 00000400:00000200:3.0:1773941497.333834:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.333842:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:3.0:1773941497.333847:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:3.0:1773941497.333852:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d1a2ec200. 00000800:00000200:3.0:1773941497.333858:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:3.0:1773941497.333864:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:3.0:1773941497.333867:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d1a2ec200 type 1, nob 616 niov 1 nkiov 1 00000400:00000200:3.0:1773941497.333885:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 02000000:00000010:3.0:1773941497.333912:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d111d1400. 00000400:00000200:3.0:1773941497.333916:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941497.333919:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccdb58. 00000400:00000010:3.0:1773941497.333926:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac110. 00000100:00000200:3.0:1773941497.333929:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291474304, portal 10 00000020:00000010:3.0:1773941497.333934:0:2400:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffff988d0646c000. 00000400:00000010:3.0:1773941497.333938:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acbb0. 00000100:00000200:3.0:1773941497.333942:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 12, xid 1860112291474304, offset 0 00000400:00000200:3.0:1773941497.333949:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941497.333954:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.333985:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000800:00000200:1.0:1773941497.333995:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941497.334002:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d1a2ec200. 00000400:00000200:1.0:1773941497.334012:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941497.334024:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941497.334029:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acd48 00000400:00000010:1.0:1773941497.334032:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acd48. 00000400:00000200:3.0:1773941497.334040:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941497.334047:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000100:00000200:1.0:1773941497.334053:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d111f2680 x1860112291474048/t0(0) o38->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 520/544 e 0 to 0 dl 1773941507 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941497.334055:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941497.334062:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941497.334074:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [93:93:255:4294967295] 192.168.204.114@tcp LPNI seq info [93:93:7:4294967295] 00000400:00000200:3.0:1773941497.334091:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.334100:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:3.0:1773941497.334103:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:3.0:1773941497.334108:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d1a2ec800. 00000800:00000200:3.0:1773941497.334115:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:3.0:1773941497.334119:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:3.0:1773941497.334121:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d1a2ec800 type 1, nob 616 niov 1 nkiov 1 00000400:00000200:3.0:1773941497.334140:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000008:00000010:3.0:1773941497.334173:0:2404:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff988d113fef90. 00020000:00000010:3.0:1773941497.334177:0:2404:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff988d20218180. 00000008:00000010:3.0:1773941497.334180:0:2404:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff988d035cba20. 00020000:00000010:3.0:1773941497.334183:0:2404:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff988d113f6400. 00000800:00000200:1.0:1773941497.334211:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941497.334217:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d1a2ec800. 00000400:00000200:1.0:1773941497.334224:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941497.334236:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941497.334255:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acbb0 00000400:00000010:1.0:1773941497.334258:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acbb0. 00000100:00000200:1.0:1773941497.334269:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d111f3800 x1860112291474304/t0(0) o38->lustre-MDT0001-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 520/544 e 0 to 0 dl 1773941507 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1773941497.335808:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941497.335815:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941497.335822:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941497.335827:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941497.335832:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941497.335836:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 10 MB=0x69bc324602a80 00000400:00000200:1.0:1773941497.335842:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941497.335849:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.114@tcp of length 416/416 into md 0x7e9 [1] + 0 00000400:00000200:1.0:1773941497.335862:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941497.335866:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:1.0:1773941497.335875:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d111f2680 x1860112291474048/t0(0) o38->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 520/544 e 0 to 0 dl 1773941507 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1773941497.335929:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccdd68. 00000400:00000200:1.0:1773941497.335936:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac220 00000400:00000010:1.0:1773941497.335938:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac220. 00000100:00000200:1.0:1773941497.335946:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d111f2680 x1860112291474048/t0(0) o38->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 520/544 e 0 to 0 dl 1773941507 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941497.335957:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d111f2680 x1860112291474048/t0(0) o38->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 520/544 e 0 to 0 dl 1773941507 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1773941497.336015:0:2400:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffff988d0352a000. 02000000:00000010:1.0:1773941497.336194:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d111d1c00. 02000000:00000010:2.0:1773941497.336196:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a4400. 02000000:00000010:1.0:1773941497.336203:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d111d2000. 00000400:00000200:2.0:1773941497.336204:0:2403:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941497.336213:0:2403:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e948. 00000100:00000010:1.0:1773941497.336213:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d111f2680. 02000000:00000010:3.0:1773941497.336219:0:14577:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d111d0c00. 00000400:00000010:2.0:1773941497.336223:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516660. 00000400:00000200:3.0:1773941497.336229:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941497.336234:0:14577:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccddc0. 00000100:00000200:2.0:1773941497.336234:0:2403:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291474176, portal 10 00000400:00000010:3.0:1773941497.336258:0:14577:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac088. 00000020:00000010:2.0:1773941497.336259:0:2403:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffff988d18b14000. 00000100:00000200:3.0:1773941497.336267:0:14577:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291474560, portal 10 00000400:00000200:1.0:1773941497.336268:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941497.336272:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045167f8. 00000400:00000200:1.0:1773941497.336278:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000010:3.0:1773941497.336279:0:14577:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac3b8. 00000100:00000200:2.0:1773941497.336281:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1860112291474176, offset 0 00000400:00000200:1.0:1773941497.336287:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000200:3.0:1773941497.336289:0:14577:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1860112291474560, offset 0 00000400:00000200:1.0:1773941497.336295:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941497.336300:0:2403:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941497.336303:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941497.336314:0:2403:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.336316:0:14577:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941497.336325:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 10 MB=0x69bc324602b80 00000400:00000200:2.0:1773941497.336326:0:2403:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941497.336332:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941497.336334:0:2403:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941497.336339:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.114@tcp of length 416/416 into md 0x7f1 [1] + 0 00000400:00000200:2.0:1773941497.336340:0:2403:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941497.336347:0:2403:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941497.336351:0:2403:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:2.0:1773941497.336357:0:2403:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [94:94:256:4294967295] 192.168.204.114@tcp LPNI seq info [94:94:8:4294967295] 00000400:00000200:2.0:1773941497.336362:0:2403:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.336370:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941497.336376:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:2.0:1773941497.336384:0:2403:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000400:00000200:3.0:1773941497.336388:0:14577:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941497.336392:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:3.0:1773941497.336405:0:14577:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941497.336412:0:14577:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000800:00000200:2.0:1773941497.336420:0:2403:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.114@tcp 00000100:00000200:1.0:1773941497.336420:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d111f3800 x1860112291474304/t0(0) o38->lustre-MDT0001-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 520/544 e 0 to 0 dl 1773941507 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941497.336421:0:14577:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941497.336432:0:14577:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000800:00000010:2.0:1773941497.336434:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d217a8a00. 00000800:00000200:2.0:1773941497.336447:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:2.0:1773941497.336454:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000400:00000010:1.0:1773941497.336460:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccdb58. 00000400:00000200:1.0:1773941497.336464:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac110 00000400:00000010:1.0:1773941497.336468:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac110. 00000800:00000200:2.0:1773941497.336470:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d217a8a00 type 1, nob 320 niov 1 nkiov 1 00000100:00000200:1.0:1773941497.336476:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d111f3800 x1860112291474304/t0(0) o38->lustre-MDT0001-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 520/544 e 0 to 0 dl 1773941507 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941497.336485:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d111f3800 x1860112291474304/t0(0) o38->lustre-MDT0001-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 520/544 e 0 to 0 dl 1773941507 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1773941497.336493:0:2403:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.336504:0:14577:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [95:95:255:4294967295] 192.168.204.114@tcp LPNI seq info [95:95:7:4294967295] 00000400:00000200:3.0:1773941497.336510:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000020:00000010:1.0:1773941497.336515:0:2400:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffff988d18bfc000. 00000400:00000200:3.0:1773941497.336527:0:14577:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:3.0:1773941497.336536:0:14577:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:3.0:1773941497.336541:0:14577:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d1a2ec800. 00000800:00000200:3.0:1773941497.336546:0:14577:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:3.0:1773941497.336551:0:14577:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:3.0:1773941497.336554:0:14577:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d1a2ec800 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:3.0:1773941497.336561:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:1.0:1773941497.336583:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941497.336588:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d217a8a00. 00000400:00000200:1.0:1773941497.336594:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941497.336603:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941497.336608:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045167f8 00000400:00000010:1.0:1773941497.336612:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045167f8. 00000100:00000200:1.0:1773941497.336617:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d111f3100 x1860112291474176/t0(0) o41->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941513 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000800:00000200:1.0:1773941497.336653:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941497.336655:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d1a2ec800. 00000400:00000200:1.0:1773941497.336658:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941497.336662:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941497.336665:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac3b8 00000400:00000010:1.0:1773941497.336666:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac3b8. 00000100:00000200:1.0:1773941497.336670:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d111f1c00 x1860112291474560/t0(0) o41->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941513 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1773941497.336825:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d111d1400. 02000000:00000010:1.0:1773941497.336830:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d111d3000. 00000100:00000010:1.0:1773941497.336833:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d111f3800. 02000000:00000010:2.0:1773941497.336842:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a5000. 00000400:00000200:2.0:1773941497.336851:0:2403:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941497.336857:0:2403:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e9a0. 00000400:00000010:2.0:1773941497.336864:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045167f8. 00000100:00000200:2.0:1773941497.336869:0:2403:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291474432, portal 10 00000020:00000010:2.0:1773941497.336877:0:2403:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffff988d18b10000. 00000400:00000010:2.0:1773941497.336884:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516908. 00000100:00000200:2.0:1773941497.336887:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1860112291474432, offset 0 00000400:00000200:2.0:1773941497.336897:0:2403:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941497.336905:0:2403:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941497.336912:0:2403:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941497.336916:0:2403:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941497.336919:0:2403:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941497.336922:0:2403:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941497.336925:0:2403:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:2.0:1773941497.336929:0:2403:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [96:96:256:4294967295] 192.168.204.114@tcp LPNI seq info [96:96:8:4294967295] 00000400:00000200:2.0:1773941497.336934:0:2403:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941497.336947:0:2403:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:2.0:1773941497.336952:0:2403:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:2.0:1773941497.336956:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d217a8b00. 00000800:00000200:2.0:1773941497.337014:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:2.0:1773941497.337022:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:2.0:1773941497.337026:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d217a8b00 type 1, nob 320 niov 1 nkiov 1 00000400:00000200:2.0:1773941497.337043:0:2403:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:1.0:1773941497.337082:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941497.337085:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d217a8b00. 00000400:00000200:1.0:1773941497.337089:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941497.337093:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941497.337096:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516908 00000400:00000010:1.0:1773941497.337098:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516908. 00000100:00000200:1.0:1773941497.337103:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d111f0380 x1860112291474432/t0(0) o41->lustre-MDT0001-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941513 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1773941497.338209:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.338218:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941497.338229:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.338235:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.338260:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.338266:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 368 into portal 10 MB=0x69bc324602b00 00000400:00000200:0.0:1773941497.338272:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.338278:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.114@tcp of length 368/368 into md 0x7f9 [1] + 224 00000400:00000200:0.0:1773941497.338304:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941497.338310:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:0.0:1773941497.338321:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d111f3100 x1860112291474176/t0(0) o41->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941513 ref 1 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1773941497.338365:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941497.338373:0:2403:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e948. 00000400:00000200:0.0:1773941497.338374:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:2.0:1773941497.338387:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516660 00000400:00000010:2.0:1773941497.338392:0:2403:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516660. 00000400:00000200:0.0:1773941497.338393:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.338405:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000200:2.0:1773941497.338413:0:2403:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d111f3100 x1860112291474176/t0(0) o41->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941513 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1773941497.338413:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.338429:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 368 into portal 10 MB=0x69bc324602c80 00000400:00000200:0.0:1773941497.338451:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.338460:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.114@tcp of length 368/368 into md 0x801 [1] + 224 00000400:00000200:0.0:1773941497.338486:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000200:2.0:1773941497.338494:0:2403:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d111f3100 x1860112291474176/t0(0) o41->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941513 ref 1 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1773941497.338504:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:0.0:1773941497.338529:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d111f1c00 x1860112291474560/t0(0) o41->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941513 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1773941497.338539:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a4400. 02000000:00000010:2.0:1773941497.338546:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff988d21fad400. 00000100:00000010:2.0:1773941497.338553:0:2403:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d111f3100. 00000400:00000010:3.0:1773941497.338576:0:14577:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccddc0. 00000400:00000200:3.0:1773941497.338582:0:14577:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac088 00000400:00000200:0.0:1773941497.338584:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941497.338586:0:14577:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac088. 00000100:00000200:3.0:1773941497.338598:0:14577:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d111f1c00 x1860112291474560/t0(0) o41->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941513 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1773941497.338600:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941497.338612:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000200:3.0:1773941497.338616:0:14577:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d111f1c00 x1860112291474560/t0(0) o41->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941513 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1773941497.338622:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.338631:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.338636:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 368 into portal 10 MB=0x69bc324602c00 00000400:00000200:0.0:1773941497.338642:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000010:3.0:1773941497.338653:0:14577:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1837d000. 00000400:00000200:0.0:1773941497.338653:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.114@tcp of length 368/368 into md 0x809 [1] + 224 02000000:00000010:3.0:1773941497.338664:0:14577:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d111d0c00. 02000000:00000010:3.0:1773941497.338669:0:14577:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff988d02cba400. 00000100:00000010:3.0:1773941497.338674:0:14577:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d111f1c00. 00800000:00000010:3.0:1773941497.338680:0:14577:0:(lmv_obd.c:1477:lmv_statfs()) kfreed 'temp': 144 at ffff988d1837d0c0. 00000100:00000010:3.0:1773941497.338689:0:14577:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d111f1c00. 02000000:00000010:3.0:1773941497.338697:0:14577:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d1817c200. 00000400:00000200:0.0:1773941497.338704:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000010:3.0:1773941497.338705:0:14577:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1837d0c0. 00000400:00000200:0.0:1773941497.338716:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 02000000:00000010:3.0:1773941497.338718:0:14577:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d111d0c00. 00000400:00000200:3.0:1773941497.338726:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000200:0.0:1773941497.338728:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d111f0380 x1860112291474432/t0(0) o41->lustre-MDT0001-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941513 ref 1 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1773941497.338729:0:14577:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccddc0. 00000400:00000010:3.0:1773941497.338735:0:14577:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac088. 00000100:00000200:3.0:1773941497.338742:0:14577:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291474688, portal 10 00000400:00000010:3.0:1773941497.338747:0:14577:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acf68. 00000100:00000200:3.0:1773941497.338757:0:14577:0:(niobuf.c:94:ptl_send_buf()) Sending 440 bytes to portal 12, xid 1860112291474688, offset 0 00000400:00000200:3.0:1773941497.338767:0:14577:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000010:2.0:1773941497.338784:0:2403:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e9a0. 00000400:00000200:2.0:1773941497.338791:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045167f8 00000400:00000010:2.0:1773941497.338794:0:2403:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045167f8. 00000100:00000200:2.0:1773941497.338805:0:2403:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d111f0380 x1860112291474432/t0(0) o41->lustre-MDT0001-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941513 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941497.338816:0:2403:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d111f0380 x1860112291474432/t0(0) o41->lustre-MDT0001-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941513 ref 1 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941497.338830:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.338840:0:14577:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 02000000:00000010:2.0:1773941497.338843:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a5000. 00000400:00000200:3.0:1773941497.338844:0:14577:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941497.338855:0:14577:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 02000000:00000010:2.0:1773941497.338857:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff988d0a0db600. 00000400:00000200:3.0:1773941497.338860:0:14577:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941497.338867:0:14577:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000100:00000010:2.0:1773941497.338870:0:2403:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d111f0380. 00000400:00000200:3.0:1773941497.338873:0:14577:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [97:97:256:4294967295] 192.168.204.114@tcp LPNI seq info [97:97:8:4294967295] 00000400:00000200:3.0:1773941497.338884:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.338901:0:14577:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:3.0:1773941497.338906:0:14577:0:(socklnd_cb.c:1001:ksocknal_send()) sending 440 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:3.0:1773941497.338911:0:14577:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d1a2ec200. 00000800:00000200:3.0:1773941497.338917:0:14577:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:3.0:1773941497.338922:0:14577:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:3.0:1773941497.338926:0:14577:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d1a2ec200 type 1, nob 536 niov 1 nkiov 1 00000400:00000200:3.0:1773941497.338942:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:0.0:1773941497.339071:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941497.339083:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d1a2ec200. 00000400:00000200:0.0:1773941497.339088:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941497.339095:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941497.339101:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acf68 00000400:00000010:0.0:1773941497.339104:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acf68. 00000100:00000200:0.0:1773941497.339114:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d111f1c00 x1860112291474688/t0(0) o40->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 440/448 e 0 to 0 dl 1773941513 ref 3 fl Rpc:r/209/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1773941497.340040:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.340055:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941497.340063:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.340071:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.340078:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.340086:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 448 into portal 10 MB=0x69bc324602d00 00000400:00000200:0.0:1773941497.340094:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.340106:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.114@tcp of length 448/448 into md 0x811 [1] + 224 00000400:00000200:0.0:1773941497.340124:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941497.340130:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:0.0:1773941497.340142:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d111f1c00 x1860112291474688/t0(0) o40->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 440/448 e 0 to 0 dl 1773941513 ref 2 fl Rpc:Qr/209/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1773941497.340188:0:14577:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccddc0. 00000400:00000200:3.0:1773941497.340194:0:14577:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac088 00000400:00000010:3.0:1773941497.340196:0:14577:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac088. 00000100:00000200:3.0:1773941497.340224:0:14577:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d111f1c00 x1860112291474688/t0(0) o40->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 440/448 e 0 to 0 dl 1773941513 ref 2 fl Rpc:RQ/209/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941497.340238:0:14577:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d111f1c00 x1860112291474688/t0(0) o40->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 440/448 e 0 to 0 dl 1773941513 ref 2 fl Rpc:RQU/209/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1773941497.340271:0:14577:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1837d0c0. 00000002:00000200:3.0:1773941497.340278:0:14577:0:(mdc_request.c:134:mdc_get_root()) root fid=[0x200000007:0x1:0x0], last_committed=0 02000000:00000010:3.0:1773941497.340284:0:14577:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d111d0c00. 02000000:00000010:3.0:1773941497.340289:0:14577:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d1817c200. 00000100:00000010:3.0:1773941497.340294:0:14577:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d111f1c00. 00000020:00000010:3.0:1773941497.340314:0:14577:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff988d035c5000. 00000020:00000010:3.0:1773941497.340344:0:14577:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff988d035c2000. 00000400:00000200:3.0:1773941497.340359:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.340367:0:14577:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000010:3.0:1773941497.340379:0:14577:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d111f1c00. 02000000:00000010:3.0:1773941497.340390:0:14577:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d111d0c00. 00000080:00000010:3.0:1773941497.340435:0:14577:0:(llite_lib.c:738:client_common_fill_super()) kmalloced '(op_data)': 384 at ffff988d1817c200. 00000020:00000010:3.0:1773941497.340440:0:14577:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 640 at ffff988d111d3000. 02000000:00000010:0.0:1773941497.340448:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d065f8800. 00000100:00000010:3.0:1773941497.340452:0:14577:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d111f0a80. 00000400:00000200:0.0:1773941497.340457:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 02000000:00000010:3.0:1773941497.340460:0:14577:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d1817ce00. 00000400:00000010:0.0:1773941497.340464:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f790. 00000100:00000010:3.0:1773941497.340468:0:14577:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1837d0c0. 00000400:00000010:0.0:1773941497.340472:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833220. 02000000:00000010:3.0:1773941497.340478:0:14577:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff988d1817c600. 00000100:00000200:0.0:1773941497.340479:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291474816, portal 4 00000400:00000200:3.0:1773941497.340485:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941497.340488:0:14577:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccddc0. 00000020:00000010:0.0:1773941497.340490:0:2400:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffff988d10804000. 00000400:00000010:3.0:1773941497.340491:0:14577:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac088. 00000100:00000200:3.0:1773941497.340498:0:14577:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112291474944, portal 10 00000400:00000010:0.0:1773941497.340503:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833aa0. 00000400:00000010:3.0:1773941497.340504:0:14577:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac6e8. 00000100:00000200:3.0:1773941497.340509:0:14577:0:(niobuf.c:94:ptl_send_buf()) Sending 264 bytes to portal 29, xid 1860112291474944, offset 0 00000100:00000200:0.0:1773941497.340512:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 28, xid 1860112291474816, offset 0 00000400:00000200:3.0:1773941497.340517:0:14577:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941497.340525:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941497.340526:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.340532:0:14577:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941497.340536:0:14577:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941497.340536:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.340540:0:14577:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941497.340544:0:14577:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941497.340549:0:14577:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:0.0:1773941497.340549:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941497.340556:0:14577:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [98:98:256:4294967295] 192.168.204.114@tcp LPNI seq info [98:98:8:4294967295] 00000400:00000200:0.0:1773941497.340559:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941497.340563:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.340575:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941497.340578:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941497.340581:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:0.0:1773941497.340585:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [99:99:256:4294967295] 192.168.204.114@tcp LPNI seq info [99:99:8:4294967295] 00000400:00000200:0.0:1773941497.340591:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.340600:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:0.0:1773941497.340607:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941497.340613:0:14577:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000010:0.0:1773941497.340615:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02d87000. 00000800:00000200:3.0:1773941497.340621:0:14577:0:(socklnd_cb.c:1001:ksocknal_send()) sending 264 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:3.0:1773941497.340626:0:14577:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d1a2ec200. 00000800:00000200:0.0:1773941497.340627:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:3.0:1773941497.340636:0:14577:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:0.0:1773941497.340641:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:3.0:1773941497.340645:0:14577:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:0.0:1773941497.340647:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02d87000 type 1, nob 616 niov 1 nkiov 1 00000400:00000200:0.0:1773941497.340663:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:3.0:1773941497.340666:0:14577:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d1a2ec200 type 1, nob 360 niov 1 nkiov 1 00000400:00000200:3.0:1773941497.340674:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:0.0:1773941497.340726:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941497.340730:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02d87000. 00000400:00000200:0.0:1773941497.340734:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941497.340740:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941497.340744:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833aa0 00000400:00000010:0.0:1773941497.340747:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833aa0. 00000100:00000200:0.0:1773941497.340756:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d111f1c00 x1860112291474816/t0(0) o8->lustre-OST0000-osc-ffff988d08f0a000@192.168.204.114@tcp:28/4 lens 520/544 e 0 to 0 dl 1773941507 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1773941497.340805:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941497.340808:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d1a2ec200. 00000400:00000200:0.0:1773941497.340813:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941497.340821:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941497.340825:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac6e8 00000400:00000010:0.0:1773941497.340828:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac6e8. 00000100:00000200:0.0:1773941497.340836:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d111f0a80 x1860112291474944/t0(0) o900->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:29/10 lens 264/248 e 0 to 0 dl 1773941513 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1773941497.341327:0:14601:0:(lproc_ptlrpc.c:1375:ldebugfs_import_seq_write()) kmalloced '(kbuf)': 34 at ffff988d05437400. 00000020:00000010:2.0:1773941497.341336:0:14601:0:(lproc_ptlrpc.c:1427:ldebugfs_import_seq_write()) kfreed 'kbuf': 34 at ffff988d05437400. 00000400:00000200:0.0:1773941497.342083:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.342088:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941497.342095:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.342103:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.342107:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.342112:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 248 into portal 10 MB=0x69bc324602e00 00000400:00000200:0.0:1773941497.342115:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.342119:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.114@tcp of length 248/248 into md 0x821 [1] + 224 00000400:00000200:0.0:1773941497.342128:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941497.342132:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:0.0:1773941497.342139:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d111f0a80 x1860112291474944/t0(0) o900->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:29/10 lens 264/248 e 0 to 0 dl 1773941513 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1773941497.342179:0:14577:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccddc0. 00000400:00000200:3.0:1773941497.342186:0:14577:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac088 00000400:00000010:3.0:1773941497.342189:0:14577:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac088. 00000100:00000200:3.0:1773941497.342200:0:14577:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d111f0a80 x1860112291474944/t0(0) o900->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:29/10 lens 264/248 e 0 to 0 dl 1773941513 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941497.342212:0:14577:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d111f0a80 x1860112291474944/t0(0) o900->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:29/10 lens 264/248 e 0 to 0 dl 1773941513 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000010:3.0:1773941497.342240:0:14577:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1837d0c0. 02000000:00000010:3.0:1773941497.342327:0:14577:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff988d1817c600. 02000000:00000010:3.0:1773941497.342333:0:14577:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d1817ce00. 00000100:00000010:3.0:1773941497.342340:0:14577:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d111f0a80. 80000000:00000010:3.0:1773941497.342351:0:14577:0:(fld_cache.c:357:fld_cache_entry_create()) kmalloced '(f_new)': 56 at ffff988d03ec7e40. 00000020:00000010:3.0:1773941497.342361:0:14577:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 640 at ffff988d111d1400. 00000100:00000010:3.0:1773941497.342370:0:14577:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d111f0a80. 02000000:00000010:3.0:1773941497.342379:0:14577:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d1817ce00. 00000100:00000010:3.0:1773941497.342390:0:14577:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1837d0c0. 02000000:00000010:3.0:1773941497.342464:0:14577:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 131072 at ffff988d18a80000. 00000400:00000200:3.0:1773941497.342470:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941497.342475:0:14577:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccddc0. 00000400:00000010:3.0:1773941497.342481:0:14577:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 632 at ffff988d111d2000. 00000100:00000200:3.0:1773941497.342487:0:14577:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 131072 bytes, xid 1860112291475072, portal 10 00000400:00000010:3.0:1773941497.342493:0:14577:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac088. 00000100:00000200:3.0:1773941497.342498:0:14577:0:(niobuf.c:94:ptl_send_buf()) Sending 448 bytes to portal 23, xid 1860112291475072, offset 0 00000400:00000200:3.0:1773941497.342510:0:14577:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941497.342516:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.342522:0:14577:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941497.342526:0:14577:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941497.342529:0:14577:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941497.342533:0:14577:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941497.342543:0:14577:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941497.342548:0:14577:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [100:100:256:4294967295] 192.168.204.114@tcp LPNI seq info [100:100:8:4294967295] 00000400:00000200:3.0:1773941497.342554:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941497.342564:0:14577:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:3.0:1773941497.342570:0:14577:0:(socklnd_cb.c:1001:ksocknal_send()) sending 448 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:3.0:1773941497.342575:0:14577:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d1a2ec800. 00000800:00000200:3.0:1773941497.342578:0:14577:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:3.0:1773941497.342582:0:14577:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:3.0:1773941497.342584:0:14577:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d1a2ec800 type 1, nob 544 niov 1 nkiov 1 00000400:00000200:3.0:1773941497.342600:0:14577:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000020:00000010:1.0:1773941497.342872:0:14607:0:(lproc_ptlrpc.c:1375:ldebugfs_import_seq_write()) kmalloced '(kbuf)': 34 at ffff988d04d67a00. 00000020:00000010:1.0:1773941497.342879:0:14607:0:(lproc_ptlrpc.c:1427:ldebugfs_import_seq_write()) kfreed 'kbuf': 34 at ffff988d04d67a00. 00000400:00000200:0.0:1773941497.343541:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.343549:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941497.343556:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.343560:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.343565:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.343572:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 4 MB=0x69bc324602d80 00000400:00000200:0.0:1773941497.343577:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941497.343583:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.204.114@tcp of length 416/416 into md 0x819 [1] + 0 00000400:00000200:0.0:1773941497.343650:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941497.343662:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:0.0:1773941497.343677:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d111f1c00 x1860112291474816/t0(0) o8->lustre-OST0000-osc-ffff988d08f0a000@192.168.204.114@tcp:28/4 lens 520/544 e 0 to 0 dl 1773941507 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000800:00000200:0.0:1773941497.343752:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941497.343756:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d1a2ec800. 00000400:00000200:0.0:1773941497.343761:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941497.343769:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941497.343773:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac088 00000400:00000010:0.0:1773941497.343775:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac088. 00000400:00000010:1.0:1773941497.343784:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f790. 00000100:00000200:0.0:1773941497.343785:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d111f0a80 x1860112291475072/t0(0) o33->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:23/10 lens 448/66264 e 0 to 0 dl 1773941513 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:0 00000400:00000200:1.0:1773941497.343791:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833220 00000400:00000010:1.0:1773941497.343796:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833220. 00000100:00000200:1.0:1773941497.343813:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d111f1c00 x1860112291474816/t0(0) o8->lustre-OST0000-osc-ffff988d08f0a000@192.168.204.114@tcp:28/4 lens 520/544 e 0 to 0 dl 1773941507 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941497.343823:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d111f1c00 x1860112291474816/t0(0) o8->lustre-OST0000-osc-ffff988d08f0a000@192.168.204.114@tcp:28/4 lens 520/544 e 0 to 0 dl 1773941507 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1773941497.343839:0:2400:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffff988d18bf8000. 02000000:00000010:1.0:1773941497.343883:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d065f8800. 02000000:00000010:1.0:1773941497.343887:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d111d0c00. 00000100:00000010:1.0:1773941497.343891:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d111f1c00. 00000020:00000010:2.0:1773941497.344061:0:14604:0:(lproc_ptlrpc.c:1375:ldebugfs_import_seq_write()) kmalloced '(kbuf)': 34 at ffff988d05437a80. 00000020:00000010:2.0:1773941497.344068:0:14604:0:(lproc_ptlrpc.c:1427:ldebugfs_import_seq_write()) kfreed 'kbuf': 34 at ffff988d05437a80. 00000400:00000200:1.0:1773941497.352707:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941497.352715:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941497.352721:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941497.352724:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941497.352729:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941497.352737:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 464 into portal 10 MB=0x69bc324602e80 00000400:00000200:1.0:1773941497.352741:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941497.352745:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.114@tcp of length 464/464 into md 0x831 [32] + 224 00000400:00000200:1.0:1773941497.352759:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941497.352762:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:1.0:1773941497.352773:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d111f0a80 x1860112291475072/t0(0) o33->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:23/10 lens 448/66264 e 0 to 0 dl 1773941513 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:0 00000400:00000010:3.0:1773941497.352821:0:14577:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccddc0. 00000400:00000200:3.0:1773941497.352827:0:14577:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d111d2000 00000400:00000010:3.0:1773941497.352830:0:14577:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 632 at ffff988d111d2000. 00000100:00000200:3.0:1773941497.352840:0:14577:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d111f0a80 x1860112291475072/t0(0) o33->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:23/10 lens 448/66264 e 0 to 0 dl 1773941513 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:0 00000100:00000200:3.0:1773941497.352849:0:14577:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d111f0a80 x1860112291475072/t0(0) o33->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:23/10 lens 448/66264 e 0 to 0 dl 1773941513 ref 2 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:0 00000100:00000010:3.0:1773941497.352874:0:14577:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1837d0c0. 00000002:00000200:3.0:1773941497.352879:0:14577:0:(mdc_request.c:172:mdc_getattr_common()) mode: 40755 00000080:00000010:3.0:1773941497.352884:0:14577:0:(llite_lib.c:753:client_common_fill_super()) kfreed 'op_data': 384 at ffff988d1817c200. 00000020:00000010:3.0:1773941497.352896:0:14577:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff988d08a3f3f0. 00000020:00000010:3.0:1773941497.352900:0:14577:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d1817c200. 00000020:00000010:3.0:1773941497.352904:0:14577:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d1817d000. 00000020:00000010:3.0:1773941497.352908:0:14577:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d209362a0. 00000008:00000010:3.0:1773941497.352914:0:14577:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff988d113f9650. 00020000:00000010:3.0:1773941497.352919:0:14577:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff988d202186c0. 00008000:00000010:3.0:1773941497.352924:0:14577:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d107f2de8. 00000020:00000010:3.0:1773941497.352928:0:14577:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d1817c600. 00000008:00000010:3.0:1773941497.352931:0:14577:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff988d035ca9a0. 00020000:00000010:3.0:1773941497.352934:0:14577:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff988d113f6880. 00008000:00000010:3.0:1773941497.352943:0:14577:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff988d209199e8. 00000080:00000010:3.0:1773941497.352949:0:14577:0:(vvp_dev.c:174:vvp_device_alloc()) kmalloced '(vdv)': 72 at ffff988d18350960. 00000080:00000010:3.0:1773941497.352954:0:14577:0:(vvp_dev.c:182:vvp_device_alloc()) kmalloced '(site)': 576 at ffff988d111d2000. 00000020:00000010:3.0:1773941497.353042:0:14577:0:(lu_object.c:1154:lu_site_init()) kmalloced '(s->ls_bkts)': 18432 at ffff988d18b78000. 00000020:00000010:3.0:1773941497.353054:0:14577:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff988d054e3800. 00000020:00000010:3.0:1773941497.353057:0:14577:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 192 at ffff988d1837d0c0. 00000008:00000010:3.0:1773941497.353083:0:14577:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff988d113fc2f0. 00020000:00000010:3.0:1773941497.353086:0:14577:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff988d20218a80. 00000080:00000010:3.0:1773941497.353096:0:14577:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff988d181e7f18. 00000080:00000010:3.0:1773941497.353100:0:14577:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff988d054ab6c0. 00000008:00000010:3.0:1773941497.353103:0:14577:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff988d035cb340. 00020000:00000010:3.0:1773941497.353106:0:14577:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff988d113f6640. 00000080:00000010:3.0:1773941497.353109:0:14577:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff988d07153000. 00000008:00000010:3.0:1773941497.353113:0:14577:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff988d113fa178. 00020000:00000010:3.0:1773941497.353115:0:14577:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff988d20218540. 00000080:00000010:3.0:1773941497.353117:0:14577:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff988d181e7498. 00000080:00000010:3.0:1773941497.353119:0:14577:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff988d054a96d0. 00000008:00000010:3.0:1773941497.353123:0:14577:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff988d035cab00. 00020000:00000010:3.0:1773941497.353126:0:14577:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff988d113f6f40. 00000080:00000010:3.0:1773941497.353128:0:14577:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff988d07153b40. 00000008:00000010:3.0:1773941497.353132:0:14577:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff988d113f8000. 00020000:00000010:3.0:1773941497.353134:0:14577:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff988d20218c00. 00000080:00000010:3.0:1773941497.353136:0:14577:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff988d181e7348. 00000080:00000010:3.0:1773941497.353138:0:14577:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff988d054a9da8. 00000008:00000010:3.0:1773941497.353140:0:14577:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff988d035ca000. 00020000:00000010:3.0:1773941497.353142:0:14577:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff988d113f4b40. 00000080:00000010:3.0:1773941497.353145:0:14577:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff988d071533c0. 00000008:00000010:3.0:1773941497.353148:0:14577:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff988d113faca0. 00020000:00000010:3.0:1773941497.353152:0:14577:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff988d20218780. 00000080:00000010:3.0:1773941497.353154:0:14577:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff988d181e71f8. 00000080:00000010:3.0:1773941497.353157:0:14577:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff988d054a8490. 00000008:00000010:3.0:1773941497.353160:0:14577:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff988d035cbb80. 00020000:00000010:3.0:1773941497.353162:0:14577:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff988d113f4d80. 00000080:00000010:3.0:1773941497.353170:0:14577:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff988d071530f0. 00020000:00000010:3.0:1773941497.353179:0:14577:0:(lovsub_dev.c:89:lovsub_device_alloc()) kmalloced '(lsd)': 72 at ffff988d183505a0. 00020000:00000010:3.0:1773941497.353185:0:14577:0:(lovsub_dev.c:89:lovsub_device_alloc()) kmalloced '(lsd)': 72 at ffff988d18350720. 00020000:00000010:3.0:1773941497.353190:0:14577:0:(lovsub_dev.c:89:lovsub_device_alloc()) kmalloced '(lsd)': 72 at ffff988d183021e0. 00000020:00000010:3.0:1773941497.353197:0:14577:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d1817d000. 00000020:00000010:3.0:1773941497.353202:0:14577:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d209362a0. 00000008:00000010:3.0:1773941497.353208:0:14577:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff988d113f9650. 00020000:00000010:3.0:1773941497.353215:0:14577:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff988d202186c0. 00008000:00000010:3.0:1773941497.353220:0:14577:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d107f2de8. 00000020:00000010:3.0:1773941497.353225:0:14577:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d1817c200. 00000008:00000010:3.0:1773941497.353229:0:14577:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff988d035ca9a0. 00020000:00000010:3.0:1773941497.353234:0:14577:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff988d113f6880. 00008000:00000010:3.0:1773941497.353238:0:14577:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff988d209199e8. 00000020:00000010:3.0:1773941497.353256:0:14577:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d1817c600. 00000020:00000010:3.0:1773941497.353260:0:14577:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff988d08a3f3f0. 00000080:00000010:3.0:1773941497.353281:0:14577:0:(super25.c:43:ll_alloc_inode()) slab-alloced 'lli': 2072 at ffff988d205298c0. 02000000:00000010:3.0:1773941497.353307:0:14577:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 131072 at ffff988d18a80000. 02000000:00000010:3.0:1773941497.353343:0:14577:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d1817ce00. 00000100:00000010:3.0:1773941497.353349:0:14577:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d111f0a80. 00000080:00000010:3.0:1773941497.353361:0:14577:0:(dcache.c:138:ll_d_init()) kmalloced '(lld)': 24 at ffff988d111d6c40. 00000080:00000010:3.0:1773941497.353370:0:14577:0:(llite_lib.c:842:client_common_fill_super()) kfreed 'data': 192 at ffff988d02eb6840. 00000080:00000010:3.0:1773941497.353375:0:14577:0:(llite_lib.c:843:client_common_fill_super()) kfreed 'osfs': 144 at ffff988d02eb69c0. 00000080:00000010:3.0:1773941497.353400:0:14577:0:(llite_lib.c:1521:ll_fill_super()) kfreed 'md': 33 at ffff988d082b2f80. 00000080:00000010:3.0:1773941497.353404:0:14577:0:(llite_lib.c:1523:ll_fill_super()) kfreed 'dt': 33 at ffff988d02f4d200. 00000080:00000010:3.0:1773941497.353409:0:14577:0:(llite_lib.c:1531:ll_fill_super()) kfreed 'cfg': 88 at ffff988d08bbb840. 00000080:02000400:3.0:1773941497.353413:0:14577:0:(llite_lib.c:1537:ll_fill_super()) Mounted lustre-client 00000400:00000001:0.0:1773941497.420217:0:14619:0:(debug.c:306:cfs_str2mask()) Process entered 00010000:00000001:3.0:1773941498.363078:0:1075:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:3.0:1773941498.363085:0:1075:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941498.363167:0:14589:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1773941498.363171:0:14589:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941498.363175:0:14589:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00000001:1.0:1773941498.589204:0:14626:0:(llite_lib.c:4092:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941498.619028:0:804:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294888936, at 4294888960 00000100:00000010:1.0:1773941498.619042:0:804:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d18350ba0. 00000100:00000040:0.0:1773941498.619111:0:247:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294888936, at 4294888960 00000100:00000010:0.0:1773941498.619119:0:247:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d183504e0. 10000000:00000001:0.0:1773941498.684363:0:14591:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941498.684370:0:14591:0:(mgc_request.c:120:config_log_put()) Process leaving 00000020:00000001:0.0:1773941498.713163:0:14647:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffb6ec819e7d08 00000080:00200008:0.0:1773941498.713172:0:14647:0:(file.c:6291:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff988d20529988), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1773941498.713179:0:14647:0:(file.c:6184:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1773941498.713180:0:14647:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffb6ec819e7c80 00000080:00200000:0.0:1773941498.713182:0:14647:0:(file.c:6186:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988),name=/ 00000080:00000010:0.0:1773941498.713187:0:14647:0:(llite_lib.c:3907:ll_prep_md_op_data()) kmalloced '(op_data)': 384 at ffff988d0381f000. 00000080:00000001:0.0:1773941498.713192:0:14647:0:(namei.c:943:ll_intent_lock()) Process entered 00000020:00000010:0.0:1773941498.713197:0:14647:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 640 at ffff988d065f9c00. 00800000:00000001:0.0:1773941498.713201:0:14647:0:(lmv_intent.c:556:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1773941498.713202:0:14647:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffb6ec819e7ba0 00800000:00000002:0.0:1773941498.713204:0:14647:0:(lmv_intent.c:563:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1773941498.713207:0:14647:0:(lmv_intent.c:437:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1773941498.713209:0:14647:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1773941498.713211:0:14647:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1773941498.713212:0:14647:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1773941498.713214:0:14647:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1773941498.713228:0:14647:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1773941498.713229:0:14647:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1773941498.713231:0:14647:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1773941498.713234:0:14647:0:(lmv_intent.c:490:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000020:00000010:0.0:1773941498.713239:0:14647:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 640 at ffff988d065fa000. 00000002:00000001:0.0:1773941498.713242:0:14647:0:(mdc_locks.c:1500:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1773941498.713243:0:14647:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffb6ec819e7a70 00000002:00010000:0.0:1773941498.713245:0:14647:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1773941498.713247:0:14647:0:(mdc_locks.c:1404:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1773941498.713249:0:14647:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:0.0:1773941498.713252:0:14647:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1773941498.713257:0:14647:0:(ldlm_lock.c:1461:ldlm_lock_match_with_skip()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1773941498.713259:0:14647:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1773941498.713260:0:14647:0:(mdc_locks.c:1456:mdc_revalidate_lock()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1773941498.713262:0:14647:0:(mdc_locks.c:1025:mdc_enqueue_base()) Process entered 00000002:00000001:0.0:1773941498.713268:0:14647:0:(mdc_locks.c:621:mdc_intent_getattr_pack()) Process entered 00000100:00000010:0.0:1773941498.713272:0:14647:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10836a00. 00000020:00000040:0.0:1773941498.713276:0:14647:0:(genops.c:1229:class_import_get()) import ffff988d035c7000 refcount=4 obd=lustre-MDT0000-mdc-ffff988d08f0a000 00010000:00000001:0.0:1773941498.713282:0:14647:0:(ldlm_request.c:853:ldlm_prep_elc_req()) Process entered 00010000:00000001:0.0:1773941498.713286:0:14647:0:(ldlm_request.c:1927:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1773941498.713289:0:14647:0:(ldlm_request.c:2099:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.713291:0:14647:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941498.713293:0:14647:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941498.713297:0:14647:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941498.713301:0:14647:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d065f8c00. 00000100:00000001:0.0:1773941498.713306:0:14647:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941498.713311:0:14647:0:(ldlm_request.c:2242:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1773941498.713312:0:14647:0:(ldlm_request.c:2245:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941498.713313:0:14647:0:(ldlm_request.c:2242:ldlm_cli_cancel_list()) Process entered 00010000:00000001:0.0:1773941498.713314:0:14647:0:(ldlm_request.c:2245:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941498.713316:0:14647:0:(ldlm_request.c:909:ldlm_prep_elc_req()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1773941498.713319:0:14647:0:(mdc_locks.c:692:mdc_intent_getattr_pack()) Process leaving (rc=18446630330367699456 : -113743341852160 : ffff988d10836a00) 00010000:00000001:0.0:1773941498.713323:0:14647:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1773941498.713325:0:14647:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1773941498.713328:0:14647:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d21ec1800. 00010000:00000010:0.0:1773941498.713331:0:14647:0:(ldlm_resource.c:1452:ldlm_resource_inodebits_new()) kmalloced '(res->lr_ibits_queues)': 112 at ffff988d181feb00. 00010000:00000001:0.0:1773941498.713336:0:14647:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1773941498.713338:0:14647:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d00030000. 00000020:00000010:0.0:1773941498.713341:0:14647:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 128 at ffff988d181fe480. 00000020:00000001:0.0:1773941498.713343:0:14647:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:0.0:1773941498.713345:0:14647:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d00030000 with handle 0x8e86767622e28c6 to hash 00000020:00000001:0.0:1773941498.713347:0:14647:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:0.0:1773941498.713348:0:14647:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:0.0:1773941498.713351:0:14647:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:0.0:1773941498.713354:0:14647:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941498.713357:0:14647:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941498.713360:0:14647:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: lustre-MDT0000-mdc-ffff988d08f0a000 lock: ffff988d00030000/0x8e86767622e28c6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x0/0x0 rrc: 2 type: IBT flags: 0x10000000000000 pid: 14647 initiator: MDT0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941498.713374:0:14647:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000 ns: lustre-MDT0000-mdc-ffff988d08f0a000 lock: ffff988d00030000/0x8e86767622e28c6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 14647 initiator: MDT0 lru_score: 0 lru_type: 0 00000020:00000010:0.0:1773941498.713383:0:14647:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffff988d10800000. 00010000:00010000:0.0:1773941498.713387:0:14647:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: lustre-MDT0000-mdc-ffff988d08f0a000 lock: ffff988d00030000/0x8e86767622e28c6 lrc: 3/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT flags: 0x0 pid: 14647 initiator: MDT0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941498.713390:0:14647:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1773941498.713392:0:14647:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941498.713397:0:14647:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d09343d80. 00000100:00000001:0.0:1773941498.713399:0:14647:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330245070208 : -113743464481408 : ffff988d09343d80) 00000100:00000001:0.0:1773941498.713401:0:14647:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941498.713402:0:14647:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367699456 : -113743341852160 : ffff988d10836a00) 00000100:00000001:0.0:1773941498.713405:0:14647:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941498.713408:0:14647:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.713410:0:14647:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941498.713411:0:14647:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941498.713419:0:14647:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10836a00 x1860112291475200/t0(0) o101->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 576/1152 e 0 to 0 dl 0 ref 2 fl New:QU/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1773941498.713426:0:14647:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941498.713427:0:14647:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941498.713430:0:14647:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941498.713432:0:14647:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941498.713434:0:14647:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941498.713437:0:14647:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10836a00 pname:cluuid:pid:xid:nid:opc:job lfs:e9ee4b2c-2bac-44e1-850a-8bf1479379ac:14647:1860112291475200:192.168.204.114@tcp:101:lfs.0 00000100:00000001:0.0:1773941498.713440:0:14647:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941498.713444:0:14647:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941498.713447:0:14647:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941498.713449:0:14647:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941498.713454:0:14647:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 2048 at ffff988d111b5800. 02000000:00000001:0.0:1773941498.713456:0:14647:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941498.713465:0:14647:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941498.713474:0:14647:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f420. 00000400:00000010:0.0:1773941498.713482:0:14647:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833aa0. 00000100:00000200:0.0:1773941498.713489:0:14647:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 2048 bytes, xid 1860112291475200, portal 10 00000100:00000001:0.0:1773941498.713491:0:14647:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941498.713492:0:14647:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367699456 : -113743341852160 : ffff988d10836a00) 00000100:00000040:0.0:1773941498.713499:0:14647:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=600 req@ffff988d10836a00 x1860112291475200/t0(0) o101->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 576/1152 e 0 to 0 dl 1773941514 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1773941498.713507:0:14647:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941498.713510:0:14647:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941498.713513:0:14647:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833b28. 00000100:00000200:0.0:1773941498.713516:0:14647:0:(niobuf.c:94:ptl_send_buf()) Sending 576 bytes to portal 12, xid 1860112291475200, offset 0 00000400:00000200:0.0:1773941498.713522:0:14647:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941498.713530:0:14647:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941498.713537:0:14647:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941498.713542:0:14647:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941498.713546:0:14647:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941498.713550:0:14647:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941498.713553:0:14647:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:0.0:1773941498.713557:0:14647:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [101:101:256:4294967295] 192.168.204.114@tcp LPNI seq info [101:101:8:4294967295] 00000400:00000200:0.0:1773941498.713562:0:14647:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941498.713572:0:14647:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:0.0:1773941498.713576:0:14647:0:(socklnd_cb.c:1001:ksocknal_send()) sending 576 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:0.0:1773941498.713581:0:14647:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02d87300. 00000800:00000200:0.0:1773941498.713586:0:14647:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:0.0:1773941498.713593:0:14647:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:0.0:1773941498.713596:0:14647:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02d87300 type 1, nob 672 niov 1 nkiov 1 00000100:00000001:0.0:1773941498.713610:0:14647:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941498.713614:0:14647:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff988d08f0a000: using a timeout of 5498 usecs (6 jiffies) 00000400:00000200:0.0:1773941498.713617:0:14647:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941498.713619:0:14647:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff988d08f0a000: PM QoS new deadline estimation for cpu 0 is 4294889060 00000100:00000010:0.0:1773941498.713622:0:14647:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580360. 00000100:00000001:0.0:1773941498.713632:0:14647:0:(niobuf.c:677:kick_cpu_latency()) lustre-MDT0000-mdc-ffff988d08f0a000: PM QoS request now active for cpu 0 00000100:00000001:0.0:1773941498.713636:0:14647:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff988d08f0a000: PM QoS new deadline estimation for cpu 1 is 4294889060 00000100:00000010:0.0:1773941498.713638:0:14647:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580240. 00000100:00000001:0.0:1773941498.713642:0:14647:0:(niobuf.c:677:kick_cpu_latency()) lustre-MDT0000-mdc-ffff988d08f0a000: PM QoS request now active for cpu 1 00000100:00000001:0.0:1773941498.713644:0:14647:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941498.713646:0:14647:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.713648:0:14647:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941498.713650:0:14647:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1773941498.713652:0:14647:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d09343d80 going to sleep for 11 seconds 00000100:00000001:0.0:1773941498.713654:0:14647:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941498.713656:0:14647:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941498.713741:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941498.713748:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02d87300. 00000400:00000200:1.0:1773941498.713753:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941498.713761:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941498.713767:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833b28 00000400:00000010:1.0:1773941498.713769:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833b28. 00000100:00000001:1.0:1773941498.713774:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941498.713779:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10836a00 x1860112291475200/t0(0) o101->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 576/1152 e 0 to 0 dl 1773941514 ref 3 fl Rpc:r/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000001:1.0:1773941498.713791:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941498.713795:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d10836a00 x1860112291475200/t0(0) o101->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 576/1152 e 0 to 0 dl 1773941514 ref 3 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000001:1.0:1773941498.713804:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.713806:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:1.0:1773941498.715940:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941498.715954:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941498.715957:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941498.715987:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941498.715995:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.716008:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941498.716016:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.716021:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.716026:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.716030:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 688 into portal 10 MB=0x69bc324602f00 00000400:00000200:1.0:1773941498.716036:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.716043:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.114@tcp of length 688/688 into md 0x839 [1] + 224 00000800:00000001:1.0:1773941498.716051:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941498.716062:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941498.716066:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941498.716071:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:1.0:1773941498.716077:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941498.716084:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10836a00 x1860112291475200/t0(0) o101->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 576/1152 e 0 to 0 dl 1773941514 ref 2 fl Rpc:Qr/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000040:1.0:1773941498.716096:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=600 mlen=688 offset=224 replen=1152 req@ffff988d10836a00 x1860112291475200/t0(0) o101->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 576/1152 e 0 to 0 dl 1773941514 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000001:1.0:1773941498.716114:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1773941498.716121:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941498.716126:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941498.716137:0:14647:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941498.716144:0:14647:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941498.716147:0:14647:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941498.716149:0:14647:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f420. 00000400:00000200:0.0:1773941498.716153:0:14647:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833aa0 00000400:00000010:0.0:1773941498.716155:0:14647:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833aa0. 00000100:00000001:0.0:1773941498.716158:0:14647:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941498.716164:0:14647:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10836a00 x1860112291475200/t0(0) o101->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 576/1152 e 0 to 0 dl 1773941514 ref 2 fl Rpc:RQ/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000200:0.0:1773941498.716176:0:14647:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10836a00 x1860112291475200/t0(0) o101->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 576/1152 e 0 to 0 dl 1773941514 ref 2 fl Rpc:RQU/600/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1773941498.716183:0:14647:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941498.716185:0:14647:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941498.716188:0:14647:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941498.716191:0:14647:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941498.716193:0:14647:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941498.716196:0:14647:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941498.716199:0:14647:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941498.716206:0:14647:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d035c7458 time=1 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941498.716209:0:14647:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d035c7410 time=1 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941498.716212:0:14647:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941498.716214:0:14647:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941498.716230:0:14647:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941498.716241:0:14647:0:(ldlm_request.c:1566:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.716245:0:14647:0:(client.c:3065:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1773941498.716247:0:14647:0:(client.c:3077:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff988d08f0a000: skip recheck: last_committed 0 00000100:00000001:0.0:1773941498.716249:0:14647:0:(client.c:3078:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1773941498.716251:0:14647:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941498.716256:0:14647:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10836a00 x1860112291475200/t0(0) o101->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 576/688 e 0 to 0 dl 1773941514 ref 2 fl Rpc:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1773941498.716264:0:14647:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941498.716266:0:14647:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941498.716268:0:14647:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1773941498.716272:0:14647:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10836a00 x1860112291475200/t0(0) o101->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 576/688 e 0 to 0 dl 1773941514 ref 2 fl Interpret:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00100000:0.0:1773941498.716281:0:14647:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10836a00 pname:cluuid:pid:xid:nid:opc:job lfs:e9ee4b2c-2bac-44e1-850a-8bf1479379ac:14647:1860112291475200:192.168.204.114@tcp:101:lfs.0 00000100:00000001:0.0:1773941498.716285:0:14647:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941498.716287:0:14647:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.716290:0:14647:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1773941498.716291:0:14647:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941498.716294:0:14647:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10836a00 x1860112291475200/t0(0) o101->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 576/688 e 0 to 0 dl 1773941514 ref 2 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1773941498.716300:0:14647:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1773941498.716303:0:14647:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d09343d80. 00000100:00000001:0.0:1773941498.716307:0:14647:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1773941498.716308:0:14647:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941498.716311:0:14647:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1773941498.716314:0:14647:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1773941498.716316:0:14647:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941498.716318:0:14647:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030000 refcount=4 00000020:00000001:0.0:1773941498.716321:0:14647:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:0.0:1773941498.716323:0:14647:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000040:0.0:1773941498.716328:0:14647:0:(ldlm_request.c:689:ldlm_cli_enqueue_fini()) local: ffff988d00030000, remote cookie: 0xd03f0241a6d949fd, flags: 0x1001 00010000:00010000:0.0:1773941498.716333:0:14647:0:(ldlm_request.c:702:ldlm_cli_enqueue_fini()) ### server returned different mode PR ns: lustre-MDT0000-mdc-ffff988d08f0a000 lock: ffff988d00030000/0x8e86767622e28c6 lrc: 4/1,0 mode: --/CR res: [0x200000007:0x1:0x0].0x0 bits 0x2/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd03f0241a6d949fd expref: -99 pid: 14647 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941498.716348:0:14647:0:(ldlm_request.c:730:ldlm_cli_enqueue_fini()) ### client-side enqueue, new policy data ns: lustre-MDT0000-mdc-ffff988d08f0a000 lock: ffff988d00030000/0x8e86767622e28c6 lrc: 4/1,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd03f0241a6d949fd expref: -99 pid: 14647 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1773941498.716355:0:14647:0:(ldlm_lock.c:1795:ldlm_lock_enqueue()) Process entered 00010000:00000001:0.0:1773941498.716359:0:14647:0:(ldlm_lock.c:1125:ldlm_grant_lock()) Process entered 00010000:00000001:0.0:1773941498.716361:0:14647:0:(ldlm_lock.c:989:search_granted_lock()) Process entered 00010000:00000001:0.0:1773941498.716363:0:14647:0:(ldlm_lock.c:1055:search_granted_lock()) Process leaving 00010000:00000001:0.0:1773941498.716364:0:14647:0:(ldlm_lock.c:1067:ldlm_granted_list_add_lock()) Process entered 00010000:00000040:0.0:1773941498.716367:0:14647:0:(ldlm_resource.c:1840:ldlm_resource_dump()) --- Resource: [0x200000007:0x1:0x0].0x0 (ffff988d21ec1800) refcount = 1 00010000:00010000:0.0:1773941498.716371:0:14647:0:(ldlm_lock.c:1072:ldlm_granted_list_add_lock()) ### About to add lock: ns: lustre-MDT0000-mdc-ffff988d08f0a000 lock: ffff988d00030000/0x8e86767622e28c6 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd03f0241a6d949fd expref: -99 pid: 14647 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1773941498.716378:0:14647:0:(ldlm_lock.c:1094:ldlm_granted_list_add_lock()) Process leaving 00000020:00000010:0.0:1773941498.716382:0:14647:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 448 at ffff988d0381e000. 00010000:00000001:0.0:1773941498.716385:0:14647:0:(ldlm_lock.c:1157:ldlm_grant_lock()) Process leaving 00010000:00000001:0.0:1773941498.716387:0:14647:0:(ldlm_lock.c:1915:ldlm_lock_enqueue()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1773941498.716390:0:14647:0:(ldlm_request.c:262:ldlm_completion_ast()) Process entered 00010000:00000001:0.0:1773941498.716393:0:14647:0:(ldlm_request.c:271:ldlm_completion_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941498.716396:0:14647:0:(ldlm_request.c:784:ldlm_cli_enqueue_fini()) ### client-side enqueue END ns: lustre-MDT0000-mdc-ffff988d08f0a000 lock: ffff988d00030000/0x8e86767622e28c6 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd03f0241a6d949fd expref: -99 pid: 14647 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1773941498.716403:0:14647:0:(ldlm_request.c:785:ldlm_cli_enqueue_fini()) Process leaving 00010000:00000001:0.0:1773941498.716405:0:14647:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941498.716407:0:14647:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941498.716408:0:14647:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941498.716409:0:14647:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941498.716410:0:14647:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1773941498.716413:0:14647:0:(mdc_locks.c:778:mdc_finish_enqueue()) Process entered 00010000:00000001:0.0:1773941498.716415:0:14647:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1773941498.716416:0:14647:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941498.716417:0:14647:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030000 refcount=3 00000020:00000001:0.0:1773941498.716418:0:14647:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:0.0:1773941498.716424:0:14647:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:0.0:1773941498.716426:0:14647:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1773941498.716427:0:14647:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941498.716428:0:14647:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030000 refcount=4 00000020:00000001:0.0:1773941498.716429:0:14647:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:0.0:1773941498.716430:0:14647:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:0.0:1773941498.716432:0:14647:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941498.716433:0:14647:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941498.716436:0:14647:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff988d08f0a000 lock: ffff988d00030000/0x8e86767622e28c6 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd03f0241a6d949fd expref: -99 pid: 14647 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1773941498.716440:0:14647:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941498.716441:0:14647:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941498.716442:0:14647:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1773941498.716443:0:14647:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941498.716444:0:14647:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030000 refcount=5 00000020:00000001:0.0:1773941498.716446:0:14647:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:0.0:1773941498.716449:0:14647:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:0.0:1773941498.716451:0:14647:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1773941498.716454:0:14647:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: lustre-MDT0000-mdc-ffff988d08f0a000 lock: ffff988d00030000/0x8e86767622e28c6 lrc: 5/2,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd03f0241a6d949fd expref: -99 pid: 14647 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1773941498.716461:0:14647:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941498.716462:0:14647:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1773941498.716465:0:14647:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-MDT0000-mdc-ffff988d08f0a000 lock: ffff988d00030000/0x8e86767622e28c6 lrc: 4/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd03f0241a6d949fd expref: -99 pid: 14647 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1773941498.716472:0:14647:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1773941498.716474:0:14647:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941498.716475:0:14647:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941498.716476:0:14647:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941498.716477:0:14647:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00100000:0.0:1773941498.716487:0:14647:0:(mdc_locks.c:833:mdc_finish_enqueue()) @@@ op=8 disposition=b, status=0 req@ffff988d10836a00 x1860112291475200/t0(0) o101->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 576/688 e 0 to 0 dl 1773941514 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00010000:00000001:0.0:1773941498.716496:0:14647:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1773941498.716498:0:14647:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941498.716499:0:14647:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030000 refcount=3 00000020:00000001:0.0:1773941498.716501:0:14647:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:0.0:1773941498.716503:0:14647:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:0.0:1773941498.716506:0:14647:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941498.716507:0:14647:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1773941498.716509:0:14647:0:(mdc_locks.c:981:mdc_finish_enqueue()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1773941498.716510:0:14647:0:(mdc_locks.c:1186:mdc_enqueue_base()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1773941498.716512:0:14647:0:(mdc_locks.c:1288:mdc_finish_intent_lock()) Process entered 00010000:00000001:0.0:1773941498.716515:0:14647:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1773941498.716515:0:14647:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941498.716516:0:14647:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030000 refcount=3 00000020:00000001:0.0:1773941498.716518:0:14647:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:0.0:1773941498.716519:0:14647:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00000002:00010000:0.0:1773941498.716522:0:14647:0:(mdc_locks.c:1357:mdc_finish_intent_lock()) ### matching against this ns: lustre-MDT0000-mdc-ffff988d08f0a000 lock: ffff988d00030000/0x8e86767622e28c6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd03f0241a6d949fd expref: -99 pid: 14647 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1773941498.716526:0:14647:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941498.716526:0:14647:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941498.716527:0:14647:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000001:0.0:1773941498.716528:0:14647:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1773941498.716529:0:14647:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941498.716530:0:14647:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030000 refcount=3 00000020:00000001:0.0:1773941498.716531:0:14647:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:0.0:1773941498.716532:0:14647:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000040:0.0:1773941498.716535:0:14647:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff988d21ec1800 count: 2 00010000:00000001:0.0:1773941498.716537:0:14647:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00010000:0.0:1773941498.716540:0:14647:0:(ldlm_lock.c:1539:ldlm_lock_match_with_skip()) ### not matched ns ffff988d203fe000 type 13 mode 4 res 8589934599/1 (0 0) 00010000:00000001:0.0:1773941498.716542:0:14647:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941498.716543:0:14647:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1773941498.716548:0:14647:0:(mdc_locks.c:1383:mdc_finish_intent_lock()) Process leaving 00000020:00000001:0.0:1773941498.716550:0:14647:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffb6ec819e79f0 00000002:00002000:0.0:1773941498.716551:0:14647:0:(mdc_locks.c:1388:mdc_finish_intent_lock()) D_IT dentry= intent=getattr status=0 disp=b: rc = 0 00000002:00000001:0.0:1773941498.716553:0:14647:0:(mdc_locks.c:1543:mdc_intent_lock()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1773941498.716555:0:14647:0:(lmv_intent.c:546:lmv_intent_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1773941498.716557:0:14647:0:(lmv_intent.c:596:lmv_intent_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00200000:0.0:1773941498.716558:0:14647:0:(namei.c:951:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 0 00000080:00000001:0.0:1773941498.716560:0:14647:0:(namei.c:989:ll_intent_lock()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1773941498.716564:0:14647:0:(llite_lib.c:4048:ll_finish_md_op_data()) kfreed 'op_data': 384 at ffff988d0381f000. 00000080:00000001:0.0:1773941498.716566:0:14647:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1773941498.716568:0:14647:0:(llite_lib.c:3689:ll_prep_inode()) Process entered 00000002:00000001:0.0:1773941498.716572:0:14647:0:(mdc_request.c:554:mdc_get_lustre_md()) Process entered 00800000:00000001:0.0:1773941498.716575:0:14647:0:(lmv_obd.c:3775:lmv_stripe_object_create()) Process entered 00800000:00000010:0.0:1773941498.716579:0:14647:0:(lmv_obd.c:3754:lmv_stripe_object_alloc()) kmalloced '(lsm_obj)': 56 at ffff988d10872600. 00800000:00000001:0.0:1773941498.716580:0:14647:0:(lmv_obd.c:3759:lmv_stripe_object_alloc()) Process leaving (rc=18446630330367944192 : -113743341607424 : ffff988d10872600) 00800000:00000001:0.0:1773941498.716582:0:14647:0:(lmv_obd.c:3836:lmv_stripe_object_create()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1773941498.716584:0:14647:0:(mdc_request.c:646:mdc_get_lustre_md()) Process leaving 00000080:00000001:0.0:1773941498.716586:0:14647:0:(llite_lib.c:1842:ll_update_lsm_md()) Process entered 00000080:00000002:0.0:1773941498.716587:0:14647:0:(llite_lib.c:1846:ll_update_lsm_md()) update lsm_obj 0000000000000000 of [0x200000007:0x1:0x0] 00000080:00000001:0.0:1773941498.716589:0:14647:0:(llite_lib.c:1797:ll_update_default_lsm_md()) Process entered 00000080:00000002:0.0:1773941498.716592:0:14647:0:(lustre_lmv.h:160:lmv_stripe_object_dump()) dump LMV: magic=0xcd30cd0 refs=1 count=1 index=4294967295 hash=invalid:0x0 max_inherit=0 max_inherit_rr=3 version=0 migrate_offset=0 migrate_hash=invalid:0 pool= 00000080:00000001:0.0:1773941498.716595:0:14647:0:(llite_lib.c:1832:ll_update_default_lsm_md()) Process leaving 00000080:00000001:0.0:1773941498.716596:0:14647:0:(llite_lib.c:1863:ll_update_lsm_md()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.716599:0:14647:0:(lcommon_cl.c:275:cl_fid_build_ino()) Process leaving (rc=144115188193296385 : 144115188193296385 : 200000007000001) 00000080:00000001:0.0:1773941498.716601:0:14647:0:(lcommon_cl.c:294:cl_fid_build_gen()) Process leaving (rc=33554432 : 33554432 : 2000000) 00000080:00200000:0.0:1773941498.716604:0:14647:0:(llite_lib.c:2979:ll_update_inode()) inode=[0x200000007:0x1:0x0], updating i_size 10752 00010000:00000001:0.0:1773941498.716605:0:14647:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1773941498.716607:0:14647:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941498.716608:0:14647:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030000 refcount=3 00000020:00000001:0.0:1773941498.716609:0:14647:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:0.0:1773941498.716610:0:14647:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:0.0:1773941498.716612:0:14647:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941498.716613:0:14647:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000080:00000001:0.0:1773941498.716614:0:14647:0:(llite_lib.c:3787:ll_prep_inode()) Process leaving via out (rc=0 : 0 : 0x0) 00800000:00000001:0.0:1773941498.716618:0:14647:0:(lmv_obd.c:4012:lmv_put_lustre_md()) Process entered 00800000:00000001:0.0:1773941498.716620:0:14647:0:(lmv_obd.c:4019:lmv_put_lustre_md()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1773941498.716622:0:14647:0:(llite_internal.h:1998:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff988d20529988) for lock 0x8e86767622e28c6 00800000:00000001:0.0:1773941498.716624:0:14647:0:(lmv_obd.c:3932:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1773941498.716625:0:14647:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1773941498.716626:0:14647:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1773941498.716627:0:14647:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941498.716628:0:14647:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030000 refcount=3 00000020:00000001:0.0:1773941498.716629:0:14647:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:0.0:1773941498.716630:0:14647:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:0.0:1773941498.716632:0:14647:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941498.716633:0:14647:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1773941498.716633:0:14647:0:(mdc_locks.c:122:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1773941498.716635:0:14647:0:(lmv_obd.c:3937:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.716636:0:14647:0:(dcache.c:242:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1773941498.716638:0:14647:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff988d20529988) 00000080:00010000:0.0:1773941498.716640:0:14647:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x8e86767622e28c6 from it ffffb6ec819e7cb8 00010000:00000001:0.0:1773941498.716641:0:14647:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1773941498.716642:0:14647:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941498.716643:0:14647:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030000 refcount=3 00000020:00000001:0.0:1773941498.716644:0:14647:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:0.0:1773941498.716645:0:14647:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:0.0:1773941498.716647:0:14647:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1773941498.716649:0:14647:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff988d08f0a000 lock: ffff988d00030000/0x8e86767622e28c6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd03f0241a6d949fd expref: -99 pid: 14647 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1773941498.716652:0:14647:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941498.716653:0:14647:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941498.716654:0:14647:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1773941498.716656:0:14647:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1773941498.716658:0:14647:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff988d08f0a000 lock: ffff988d00030000/0x8e86767622e28c6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd03f0241a6d949fd expref: -99 pid: 14647 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1773941498.716664:0:14647:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1773941498.716668:0:14647:0:(ldlm_request.c:2131:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1773941498.716669:0:14647:0:(ldlm_request.c:1927:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1773941498.716672:0:14647:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941498.716673:0:14647:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941498.716674:0:14647:0:(ldlm_request.c:2099:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941498.716676:0:14647:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1773941498.716678:0:14647:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941498.716679:0:14647:0:(ldlm_request.c:2141:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941498.716681:0:14647:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941498.716682:0:14647:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1773941498.716683:0:14647:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941498.716684:0:14647:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1773941498.716685:0:14647:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941498.716688:0:14647:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10836a00 x1860112291475200/t0(0) o101->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 576/688 e 0 to 0 dl 1773941514 ref 1 fl Complete:RQU/600/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:0 00000100:00000001:0.0:1773941498.716693:0:14647:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1773941498.716694:0:14647:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1773941498.716697:0:14647:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 2048 at ffff988d111b5800. 02000000:00000001:0.0:1773941498.716699:0:14647:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1773941498.716700:0:14647:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941498.716702:0:14647:0:(genops.c:1242:class_import_put()) import ffff988d035c7000 refcount=3 obd=lustre-MDT0000-mdc-ffff988d08f0a000 00000020:00000001:0.0:1773941498.716703:0:14647:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1773941498.716705:0:14647:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d065f8c00. 02000000:00000001:0.0:1773941498.716708:0:14647:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1773941498.716710:0:14647:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1773941498.716711:0:14647:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10836a00. 00000100:00000001:0.0:1773941498.716713:0:14647:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941498.716714:0:14647:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1773941498.716717:0:14647:0:(file.c:6244:ll_merge_md_attr()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1773941498.716721:0:14647:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 1856 at ffff988d111b5800. 00000020:00000001:0.0:1773941498.716723:0:14647:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffb6ec819e7d08 00000080:00000008:0.0:1773941498.716724:0:14647:0:(file.c:6469:ll_getattr_dentry()) COMPLETED file /:[0x200000007:0x1:0x0](ffff988d20529988), request_mask 2047, flags 0, foreign 0 00000080:00000001:0.0:1773941498.717295:0:14647:0:(llite_lib.c:4092:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941498.717322:0:14647:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffb6ec819e7d08 00000080:00200008:0.0:1773941498.717325:0:14647:0:(file.c:6291:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff988d20529988), request_mask 0, flags 0, foreign 0 00000080:00000001:0.0:1773941498.717329:0:14647:0:(file.c:6184:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1773941498.717336:0:14647:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffb6ec819e7c80 00000080:00200000:0.0:1773941498.717338:0:14647:0:(file.c:6186:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988),name=/ 00000080:00000010:0.0:1773941498.717342:0:14647:0:(llite_lib.c:3907:ll_prep_md_op_data()) kmalloced '(op_data)': 384 at ffff988d0381f000. 00800000:00000002:0.0:1773941498.717346:0:14647:0:(lmv_obd.c:3847:lmv_stripe_object_get()) get ffff988d10872600 2 00000080:00000001:0.0:1773941498.717348:0:14647:0:(namei.c:943:ll_intent_lock()) Process entered 00800000:00000001:0.0:1773941498.717350:0:14647:0:(lmv_intent.c:556:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1773941498.717351:0:14647:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffb6ec819e7ba0 00800000:00000002:0.0:1773941498.717352:0:14647:0:(lmv_intent.c:563:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1773941498.717354:0:14647:0:(lmv_intent.c:437:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1773941498.717355:0:14647:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1773941498.717356:0:14647:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1773941498.717358:0:14647:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1773941498.717359:0:14647:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1773941498.717360:0:14647:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1773941498.717362:0:14647:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1773941498.717363:0:14647:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1773941498.717364:0:14647:0:(lmv_intent.c:490:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1773941498.717367:0:14647:0:(mdc_locks.c:1500:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1773941498.717368:0:14647:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffb6ec819e7a70 00000002:00010000:0.0:1773941498.717369:0:14647:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1773941498.717371:0:14647:0:(mdc_locks.c:1404:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1773941498.717372:0:14647:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:0.0:1773941498.717373:0:14647:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1773941498.717376:0:14647:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff988d21ec1800 count: 2 00010000:00000001:0.0:1773941498.717377:0:14647:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1773941498.717379:0:14647:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941498.717381:0:14647:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1773941498.717383:0:14647:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff988d08f0a000 lock: ffff988d00030000/0x8e86767622e28c6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd03f0241a6d949fd expref: -99 pid: 14647 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 02000000:00000001:0.0:1773941498.717389:0:14647:0:(sec.c:1015:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1773941498.717391:0:14647:0:(sec.c:1031:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941498.717393:0:14647:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff988d08f0a000 lock: ffff988d00030000/0x8e86767622e28c6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd03f0241a6d949fd expref: -99 pid: 14647 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000002:00000001:0.0:1773941498.717404:0:14647:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1773941498.717405:0:14647:0:(mdc_locks.c:1456:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1773941498.717406:0:14647:0:(mdc_locks.c:1523:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1773941498.717407:0:14647:0:(lmv_intent.c:511:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1773941498.717409:0:14647:0:(lmv_intent.c:596:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1773941498.717410:0:14647:0:(namei.c:951:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00000001:0.0:1773941498.717411:0:14647:0:(namei.c:989:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1773941498.717413:0:14647:0:(lmv_obd.c:3892:lmv_stripe_object_put()) put ffff988d10872600 2 00000080:00000010:0.0:1773941498.717415:0:14647:0:(llite_lib.c:4048:ll_finish_md_op_data()) kfreed 'op_data': 384 at ffff988d0381f000. 00000080:00000001:0.0:1773941498.717416:0:14647:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1773941498.717417:0:14647:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1773941498.717419:0:14647:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff988d20529988) 00000080:00010000:0.0:1773941498.717421:0:14647:0:(llite_internal.h:1998:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff988d20529988) for lock 0x8e86767622e28c6 00800000:00000001:0.0:1773941498.717422:0:14647:0:(lmv_obd.c:3932:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1773941498.717424:0:14647:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1773941498.717424:0:14647:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1773941498.717425:0:14647:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941498.717426:0:14647:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030000 refcount=3 00000020:00000001:0.0:1773941498.717428:0:14647:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:0.0:1773941498.717429:0:14647:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:0.0:1773941498.717430:0:14647:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941498.717431:0:14647:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1773941498.717432:0:14647:0:(mdc_locks.c:122:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1773941498.717433:0:14647:0:(lmv_obd.c:3937:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1773941498.717435:0:14647:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x8e86767622e28c6 from it ffffb6ec819e7cb8 00010000:00000001:0.0:1773941498.717436:0:14647:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1773941498.717437:0:14647:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941498.717438:0:14647:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030000 refcount=3 00000020:00000001:0.0:1773941498.717439:0:14647:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:0.0:1773941498.717440:0:14647:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:0.0:1773941498.717441:0:14647:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1773941498.717447:0:14647:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff988d08f0a000 lock: ffff988d00030000/0x8e86767622e28c6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd03f0241a6d949fd expref: -99 pid: 14647 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00000001:0.0:1773941498.717454:0:14647:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941498.717455:0:14647:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941498.717456:0:14647:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1773941498.717458:0:14647:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1773941498.717461:0:14647:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff988d08f0a000 lock: ffff988d00030000/0x8e86767622e28c6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd03f0241a6d949fd expref: -99 pid: 14647 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00010000:00000001:0.0:1773941498.717467:0:14647:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1773941498.717469:0:14647:0:(ldlm_request.c:2131:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1773941498.717470:0:14647:0:(ldlm_request.c:1927:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1773941498.717472:0:14647:0:(ldlm_request.c:2099:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941498.717474:0:14647:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1773941498.717476:0:14647:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941498.717478:0:14647:0:(ldlm_request.c:2141:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941498.717480:0:14647:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941498.717482:0:14647:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1773941498.717483:0:14647:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941498.717484:0:14647:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1773941498.717486:0:14647:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941498.717488:0:14647:0:(client.c:2844:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:0.0:1773941498.717490:0:14647:0:(file.c:6244:ll_merge_md_attr()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941498.717493:0:14647:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffb6ec819e7d08 00000080:00000008:0.0:1773941498.717495:0:14647:0:(file.c:6469:ll_getattr_dentry()) COMPLETED file /:[0x200000007:0x1:0x0](ffff988d20529988), request_mask 0, flags 0, foreign 0 00000080:00000001:0.0:1773941498.717559:0:14647:0:(file.c:6653:ll_inode_permission()) Process entered 00000080:00000001:0.0:1773941498.717560:0:14647:0:(file.c:6184:ll_inode_revalidate()) Process entered 00000020:00000001:0.0:1773941498.717562:0:14647:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffb6ec819e7b78 00000080:00200000:0.0:1773941498.717563:0:14647:0:(file.c:6186:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988),name=/ 00000080:00000010:0.0:1773941498.717566:0:14647:0:(llite_lib.c:3907:ll_prep_md_op_data()) kmalloced '(op_data)': 384 at ffff988d0381f000. 00800000:00000002:0.0:1773941498.717568:0:14647:0:(lmv_obd.c:3847:lmv_stripe_object_get()) get ffff988d10872600 2 00000080:00000001:0.0:1773941498.717569:0:14647:0:(namei.c:943:ll_intent_lock()) Process entered 00800000:00000001:0.0:1773941498.717571:0:14647:0:(lmv_intent.c:556:lmv_intent_lock()) Process entered 00000020:00000001:0.0:1773941498.717572:0:14647:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffb6ec819e7a98 00800000:00000002:0.0:1773941498.717573:0:14647:0:(lmv_intent.c:563:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:0.0:1773941498.717578:0:14647:0:(lmv_intent.c:437:lmv_intent_lookup()) Process entered 00800000:00000001:0.0:1773941498.717579:0:14647:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:0.0:1773941498.717580:0:14647:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:0.0:1773941498.717581:0:14647:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:0.0:1773941498.717582:0:14647:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:0.0:1773941498.717583:0:14647:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1773941498.717584:0:14647:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:0.0:1773941498.717586:0:14647:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:0.0:1773941498.717587:0:14647:0:(lmv_intent.c:490:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:0.0:1773941498.717590:0:14647:0:(mdc_locks.c:1500:mdc_intent_lock()) Process entered 00000020:00000001:0.0:1773941498.717591:0:14647:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffb6ec819e7968 00000002:00010000:0.0:1773941498.717592:0:14647:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:0.0:1773941498.717594:0:14647:0:(mdc_locks.c:1404:mdc_revalidate_lock()) Process entered 00000002:00000001:0.0:1773941498.717595:0:14647:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:0.0:1773941498.717596:0:14647:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:0.0:1773941498.717598:0:14647:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff988d21ec1800 count: 2 00010000:00000001:0.0:1773941498.717600:0:14647:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1773941498.717602:0:14647:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941498.717603:0:14647:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1773941498.717606:0:14647:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff988d08f0a000 lock: ffff988d00030000/0x8e86767622e28c6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd03f0241a6d949fd expref: -99 pid: 14647 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 02000000:00000001:0.0:1773941498.717610:0:14647:0:(sec.c:1015:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:0.0:1773941498.717612:0:14647:0:(sec.c:1031:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941498.717614:0:14647:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff988d08f0a000 lock: ffff988d00030000/0x8e86767622e28c6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd03f0241a6d949fd expref: -99 pid: 14647 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00000002:00000001:0.0:1773941498.717618:0:14647:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:0.0:1773941498.717619:0:14647:0:(mdc_locks.c:1456:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:0.0:1773941498.717620:0:14647:0:(mdc_locks.c:1523:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1773941498.717621:0:14647:0:(lmv_intent.c:511:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:0.0:1773941498.717622:0:14647:0:(lmv_intent.c:596:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1773941498.717624:0:14647:0:(namei.c:951:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00000001:0.0:1773941498.717625:0:14647:0:(namei.c:989:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:0.0:1773941498.717629:0:14647:0:(lmv_obd.c:3892:lmv_stripe_object_put()) put ffff988d10872600 2 00000080:00000010:0.0:1773941498.717631:0:14647:0:(llite_lib.c:4048:ll_finish_md_op_data()) kfreed 'op_data': 384 at ffff988d0381f000. 00000080:00000001:0.0:1773941498.717633:0:14647:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:0.0:1773941498.717634:0:14647:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1773941498.717635:0:14647:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff988d20529988) 00000080:00010000:0.0:1773941498.717637:0:14647:0:(llite_internal.h:1998:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff988d20529988) for lock 0x8e86767622e28c6 00800000:00000001:0.0:1773941498.717639:0:14647:0:(lmv_obd.c:3932:lmv_set_lock_data()) Process entered 00000002:00000001:0.0:1773941498.717640:0:14647:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:0.0:1773941498.717641:0:14647:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1773941498.717642:0:14647:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941498.717643:0:14647:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030000 refcount=3 00000020:00000001:0.0:1773941498.717644:0:14647:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:0.0:1773941498.717645:0:14647:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:0.0:1773941498.717647:0:14647:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941498.717647:0:14647:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:0.0:1773941498.717648:0:14647:0:(mdc_locks.c:122:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1773941498.717649:0:14647:0:(lmv_obd.c:3937:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:0.0:1773941498.717651:0:14647:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x8e86767622e28c6 from it ffffb6ec819e7bb0 00010000:00000001:0.0:1773941498.717652:0:14647:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1773941498.717653:0:14647:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941498.717654:0:14647:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030000 refcount=3 00000020:00000001:0.0:1773941498.717655:0:14647:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:0.0:1773941498.717656:0:14647:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:0.0:1773941498.717658:0:14647:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1773941498.717660:0:14647:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff988d08f0a000 lock: ffff988d00030000/0x8e86767622e28c6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd03f0241a6d949fd expref: -99 pid: 14647 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 1 00010000:00000001:0.0:1773941498.717664:0:14647:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941498.717665:0:14647:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941498.717666:0:14647:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:0.0:1773941498.717667:0:14647:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:0.0:1773941498.717669:0:14647:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff988d08f0a000 lock: ffff988d00030000/0x8e86767622e28c6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd03f0241a6d949fd expref: -99 pid: 14647 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00010000:00000001:0.0:1773941498.717681:0:14647:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1773941498.717683:0:14647:0:(ldlm_request.c:2131:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1773941498.717685:0:14647:0:(ldlm_request.c:1927:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1773941498.717687:0:14647:0:(ldlm_request.c:2099:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941498.717690:0:14647:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1773941498.717691:0:14647:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941498.717693:0:14647:0:(ldlm_request.c:2141:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941498.717696:0:14647:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941498.717698:0:14647:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1773941498.717699:0:14647:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941498.717701:0:14647:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1773941498.717702:0:14647:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941498.717704:0:14647:0:(client.c:2844:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:0.0:1773941498.717706:0:14647:0:(file.c:6668:ll_inode_permission()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988), inode mode 41ed mask 44 00000080:00000001:0.0:1773941498.717710:0:14647:0:(file.c:6707:ll_inode_permission()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.717714:0:14647:0:(dir.c:2974:ll_dir_open()) Process entered 00000080:00000001:0.0:1773941498.717716:0:14647:0:(file.c:1021:ll_file_open()) Process entered 00000020:00000001:0.0:1773941498.717719:0:14647:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffb6ec819e7b78 00000080:00200008:0.0:1773941498.717721:0:14647:0:(file.c:1025:ll_file_open()) START file /:[0x200000007:0x1:0x0](ffff988d20529988), flags 100000 00000080:00000010:0.0:1773941498.717726:0:14647:0:(file.c:67:ll_file_data_get()) slab-alloced 'lfd': 288 at ffff988d0a346140. 00000080:00000001:0.0:1773941498.717730:0:14647:0:(file.c:1050:ll_file_open()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.717732:0:14647:0:(dir.c:2975:ll_dir_open()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.717800:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.717803:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.717808:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.717810:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.717814:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.717816:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.717818:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.717820:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.717822:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.717827:0:14647:0:(obd_class.h:1067:obd_statfs()) Process entered 00800000:00000004:0.0:1773941498.717829:0:14647:0:(obd_class.h:1083:obd_statfs()) lustre-MDT0000-mdc-ffff988d08f0a000: age 220, max_age 220 00800000:00000004:0.0:1773941498.717831:0:14647:0:(obd_class.h:1123:obd_statfs()) lustre-MDT0000-mdc-ffff988d08f0a000: use ffff988d089ab7c0 cache blocks 2956/2997 objects 86559/86890 00800000:00000001:0.0:1773941498.717833:0:14647:0:(obd_class.h:1128:obd_statfs()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1773941498.717839:0:14647:0:(lmv_obd.c:1164:lmv_iocontrol()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.717841:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1773941498.717842:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.717845:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.717846:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.717860:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.717861:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.717863:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.717864:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.717866:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.717868:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.717869:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.717870:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.717871:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.717874:0:14647:0:(obd_class.h:1067:obd_statfs()) Process entered 00800000:00000004:0.0:1773941498.717875:0:14647:0:(obd_class.h:1083:obd_statfs()) lustre-MDT0001-mdc-ffff988d08f0a000: age -780, max_age 220 00000002:00000001:0.0:1773941498.717877:0:14647:0:(mdc_request.c:1658:mdc_statfs()) Process entered 00000020:00000040:0.0:1773941498.717879:0:14647:0:(genops.c:1229:class_import_get()) import ffff988d035c6000 refcount=4 obd=lustre-MDT0001-mdc-ffff988d08f0a000 00000100:00000010:0.0:1773941498.717882:0:14647:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10836a00. 00000020:00000040:0.0:1773941498.717884:0:14647:0:(genops.c:1229:class_import_get()) import ffff988d035c6000 refcount=5 obd=lustre-MDT0001-mdc-ffff988d08f0a000 00000100:00000001:0.0:1773941498.717887:0:14647:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941498.717889:0:14647:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941498.717891:0:14647:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941498.717895:0:14647:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff988d02d87300. 00000100:00000001:0.0:1773941498.717898:0:14647:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.717900:0:14647:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1773941498.717902:0:14647:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941498.717905:0:14647:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d09343d80. 00000100:00000001:0.0:1773941498.717906:0:14647:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330245070208 : -113743464481408 : ffff988d09343d80) 00000100:00000001:0.0:1773941498.717908:0:14647:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941498.717909:0:14647:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367699456 : -113743341852160 : ffff988d10836a00) 00000100:00000001:0.0:1773941498.717911:0:14647:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941498.717913:0:14647:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.717915:0:14647:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941498.717916:0:14647:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941498.717920:0:14647:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10836a00 x1860112291475328/t0(0) o41->lustre-MDT0001-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941498.717930:0:14647:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941498.717931:0:14647:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941498.717934:0:14647:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941498.717935:0:14647:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941498.717937:0:14647:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941498.717941:0:14647:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10836a00 pname:cluuid:pid:xid:nid:opc:job lfs:e9ee4b2c-2bac-44e1-850a-8bf1479379ac:14647:1860112291475328:192.168.204.114@tcp:41:lfs.0 00000100:00000001:0.0:1773941498.717944:0:14647:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941498.717947:0:14647:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941498.717948:0:14647:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941498.717950:0:14647:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941498.717953:0:14647:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d065f8000. 02000000:00000001:0.0:1773941498.717955:0:14647:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941498.717959:0:14647:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941498.717989:0:14647:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f420. 00000400:00000010:0.0:1773941498.717993:0:14647:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833aa0. 00000100:00000200:0.0:1773941498.717996:0:14647:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291475328, portal 10 00000100:00000001:0.0:1773941498.717998:0:14647:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941498.717999:0:14647:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367699456 : -113743341852160 : ffff988d10836a00) 00000020:00000010:0.0:1773941498.718004:0:14647:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 4608 at ffff988d10802000. 00000100:00000040:0.0:1773941498.718010:0:14647:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10836a00 x1860112291475328/t0(0) o41->lustre-MDT0001-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941514 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941498.718015:0:14647:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941498.718017:0:14647:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941498.718019:0:14647:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d108332a8. 00000100:00000200:0.0:1773941498.718021:0:14647:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1860112291475328, offset 0 00000400:00000200:0.0:1773941498.718024:0:14647:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941498.718027:0:14647:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941498.718031:0:14647:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941498.718035:0:14647:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941498.718038:0:14647:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941498.718040:0:14647:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941498.718046:0:14647:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:0.0:1773941498.718049:0:14647:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [102:102:256:4294967295] 192.168.204.114@tcp LPNI seq info [102:102:8:4294967295] 00000400:00000200:0.0:1773941498.718052:0:14647:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941498.718059:0:14647:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:0.0:1773941498.718063:0:14647:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:0.0:1773941498.718068:0:14647:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d04c1a600. 00000800:00000200:0.0:1773941498.718073:0:14647:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:0.0:1773941498.718078:0:14647:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:0.0:1773941498.718081:0:14647:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d04c1a600 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1773941498.718099:0:14647:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941498.718102:0:14647:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff988d08f0a000: using a timeout of 5062 usecs (6 jiffies) 00000400:00000200:0.0:1773941498.718105:0:14647:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941498.718107:0:14647:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff988d08f0a000: PM QoS new deadline estimation for cpu 0 is 4294889065 00000100:00000001:0.0:1773941498.718112:0:14647:0:(niobuf.c:684:kick_cpu_latency()) lustre-MDT0001-mdc-ffff988d08f0a000: PM QoS request active for cpu 0, simply extend its deadline from 4294889060 00000100:00000001:0.0:1773941498.718117:0:14647:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff988d08f0a000: PM QoS new deadline estimation for cpu 1 is 4294889065 00000100:00000001:0.0:1773941498.718121:0:14647:0:(niobuf.c:684:kick_cpu_latency()) lustre-MDT0001-mdc-ffff988d08f0a000: PM QoS request active for cpu 1, simply extend its deadline from 4294889060 00000100:00000001:0.0:1773941498.718125:0:14647:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941498.718128:0:14647:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.718131:0:14647:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941498.718133:0:14647:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1773941498.718135:0:14647:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d09343d80 going to sleep for 11 seconds 00000100:00000001:0.0:1773941498.718138:0:14647:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941498.718140:0:14647:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941498.718162:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941498.718167:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d04c1a600. 00000400:00000200:1.0:1773941498.718171:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941498.718176:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941498.718179:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d108332a8 00000400:00000010:1.0:1773941498.718181:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d108332a8. 00000100:00000001:1.0:1773941498.718184:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941498.718189:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10836a00 x1860112291475328/t0(0) o41->lustre-MDT0001-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941514 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.718197:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941498.718202:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d10836a00 x1860112291475328/t0(0) o41->lustre-MDT0001-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941514 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.718235:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.718237:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:1.0:1773941498.719202:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941498.719209:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941498.719211:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941498.719212:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941498.719231:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.719235:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941498.719240:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.719243:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.719245:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.719248:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 368 into portal 10 MB=0x69bc324602f80 00000400:00000200:1.0:1773941498.719251:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.719254:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.114@tcp of length 368/368 into md 0x849 [1] + 224 00000800:00000001:1.0:1773941498.719258:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941498.719263:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941498.719265:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941498.719267:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:1.0:1773941498.719270:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941498.719273:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10836a00 x1860112291475328/t0(0) o41->lustre-MDT0001-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941514 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941498.719281:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=368 offset=224 replen=368 req@ffff988d10836a00 x1860112291475328/t0(0) o41->lustre-MDT0001-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941514 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.719293:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1773941498.719296:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941498.719298:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941498.719312:0:14647:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941498.719316:0:14647:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941498.719317:0:14647:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941498.719319:0:14647:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f420. 00000400:00000200:0.0:1773941498.719322:0:14647:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833aa0 00000400:00000010:0.0:1773941498.719323:0:14647:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833aa0. 00000100:00000001:0.0:1773941498.719325:0:14647:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941498.719329:0:14647:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10836a00 x1860112291475328/t0(0) o41->lustre-MDT0001-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941514 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941498.719341:0:14647:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10836a00 x1860112291475328/t0(0) o41->lustre-MDT0001-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941514 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941498.719346:0:14647:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941498.719347:0:14647:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941498.719348:0:14647:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941498.719350:0:14647:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941498.719351:0:14647:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941498.719352:0:14647:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941498.719354:0:14647:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941498.719358:0:14647:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d035c6458 time=1 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941498.719360:0:14647:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d035c6410 time=1 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941498.719361:0:14647:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941498.719362:0:14647:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941498.719364:0:14647:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1773941498.719367:0:14647:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff988d10836a00 x1860112291475328/t0(0) o41->lustre-MDT0001-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941514 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1773941498.719372:0:14647:0:(ldlm_request.c:1535:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.719373:0:14647:0:(client.c:3065:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1773941498.719374:0:14647:0:(client.c:3077:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff988d08f0a000: skip recheck: last_committed 0 00000100:00000001:0.0:1773941498.719375:0:14647:0:(client.c:3078:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1773941498.719376:0:14647:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941498.719378:0:14647:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10836a00 x1860112291475328/t0(0) o41->lustre-MDT0001-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941514 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941498.719382:0:14647:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941498.719384:0:14647:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941498.719385:0:14647:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1773941498.719387:0:14647:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10836a00 x1860112291475328/t0(0) o41->lustre-MDT0001-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941514 ref 2 fl Interpret:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941498.719392:0:14647:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10836a00 pname:cluuid:pid:xid:nid:opc:job lfs:e9ee4b2c-2bac-44e1-850a-8bf1479379ac:14647:1860112291475328:192.168.204.114@tcp:41:lfs.0 00000100:00000001:0.0:1773941498.719395:0:14647:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941498.719399:0:14647:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.719401:0:14647:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1773941498.719402:0:14647:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941498.719404:0:14647:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10836a00 x1860112291475328/t0(0) o41->lustre-MDT0001-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941514 ref 2 fl Complete:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941498.719408:0:14647:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1773941498.719410:0:14647:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d09343d80. 00000100:00000001:0.0:1773941498.719412:0:14647:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1773941498.719413:0:14647:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:0.0:1773941498.719416:0:14647:0:(mdc_request.c:1710:mdc_statfs()) Process leaving 00000100:00000001:0.0:1773941498.719416:0:14647:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941498.719418:0:14647:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10836a00 x1860112291475328/t0(0) o41->lustre-MDT0001-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/368 e 0 to 0 dl 1773941514 ref 1 fl Complete:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941498.719423:0:14647:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1773941498.719424:0:14647:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1773941498.719425:0:14647:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d065f8000. 02000000:00000001:0.0:1773941498.719427:0:14647:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1773941498.719428:0:14647:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941498.719428:0:14647:0:(genops.c:1242:class_import_put()) import ffff988d035c6000 refcount=4 obd=lustre-MDT0001-mdc-ffff988d08f0a000 00000020:00000001:0.0:1773941498.719430:0:14647:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1773941498.719431:0:14647:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff988d02d87300. 02000000:00000001:0.0:1773941498.719433:0:14647:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1773941498.719434:0:14647:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1773941498.719435:0:14647:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10836a00. 00000100:00000001:0.0:1773941498.719437:0:14647:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941498.719438:0:14647:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1773941498.719439:0:14647:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941498.719440:0:14647:0:(genops.c:1242:class_import_put()) import ffff988d035c6000 refcount=3 obd=lustre-MDT0001-mdc-ffff988d08f0a000 00000020:00000001:0.0:1773941498.719441:0:14647:0:(genops.c:1249:class_import_put()) Process leaving 00800000:00000004:0.0:1773941498.719442:0:14647:0:(obd_class.h:1109:obd_statfs()) lustre-MDT0001-mdc-ffff988d08f0a000: update ffff988d089ad178 cache blocks 2957/2997 objects 86774/87086 00800000:00000001:0.0:1773941498.719444:0:14647:0:(obd_class.h:1128:obd_statfs()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:0.0:1773941498.719446:0:14647:0:(lmv_obd.c:1164:lmv_iocontrol()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.719447:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1773941498.719449:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.719450:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.719451:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.719475:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.719476:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.719479:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.719481:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.719483:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.719484:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.719486:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.719487:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.719488:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.719491:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719493:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719494:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.719497:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.719499:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719500:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719502:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.719504:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.719506:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.719506:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.719508:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.719509:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.719510:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.719512:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.719513:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.719515:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719516:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719517:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.719518:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.719520:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719521:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719523:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.719524:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.719526:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.719530:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.719532:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.719533:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.719534:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.719535:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.719536:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.719538:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719539:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719541:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.719542:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.719544:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719545:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719548:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.719549:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.719550:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.719551:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.719560:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.719562:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.719566:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.719567:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.719568:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.719569:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719570:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719572:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.719573:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.719575:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719577:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719579:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.719580:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.719582:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.719583:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.719585:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.719586:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.719590:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.719592:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.719593:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.719595:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719597:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719598:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.719599:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.719602:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719603:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719606:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.719608:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.719610:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.719611:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.719613:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.719615:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.719616:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.719617:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.719618:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.719620:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719621:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719622:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.719624:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.719626:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719627:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719629:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.719629:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.719631:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.719631:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.719633:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.719635:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.719636:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.719637:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.719638:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.719643:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719645:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719647:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.719648:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.719650:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719651:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719654:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.719655:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.719656:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.719657:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.719659:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.719661:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.719662:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.719663:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.719664:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.719666:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719667:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719668:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.719669:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.719671:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719673:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719675:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.719677:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.719678:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.719679:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.719681:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.719683:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.719684:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.719685:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.719686:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.719687:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719688:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719693:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.719694:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.719696:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719697:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719700:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.719701:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.719703:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.719705:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.719707:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.719708:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.719709:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.719710:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.719711:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.719712:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719713:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719714:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.719716:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.719717:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719719:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719721:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.719722:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.719723:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.719724:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.719726:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.719727:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.719728:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.719729:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.719730:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.719732:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719733:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719734:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.719736:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.719740:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719741:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719744:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.719745:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.719747:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.719747:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.719749:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.719750:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.719751:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.719752:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.719752:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.719754:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719755:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719756:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.719757:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.719758:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719760:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719761:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.719762:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.719763:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.719764:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.719765:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.719766:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.719767:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.719768:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.719769:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.719770:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719771:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719772:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.719773:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.719775:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719776:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719777:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.719781:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.719783:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.719783:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.719785:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.719786:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.719787:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.719788:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.719789:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.719790:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719791:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719792:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.719794:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.719795:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719796:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719798:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.719799:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.719801:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.719802:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.719804:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.719806:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.719808:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.719808:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.719810:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.719812:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719814:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719816:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.719818:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.719820:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719821:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719824:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.719826:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.719828:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.719830:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.719841:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.719843:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.719845:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.719846:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.719848:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.719850:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719852:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719854:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.719856:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.719858:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719860:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719865:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.719866:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.719868:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.719869:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.719872:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.719874:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.719876:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.719877:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.719878:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.719880:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719882:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719884:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.719886:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.719888:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719890:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719892:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.719894:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.719896:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.719897:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.719899:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.719901:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.719903:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.719909:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.719910:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.719913:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719915:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719916:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.719918:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.719920:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719922:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719926:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.719927:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.719930:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.719931:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.719933:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.719936:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.719937:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.719938:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.719940:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.719943:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719944:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719946:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.719948:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.719950:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719951:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.719954:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.719955:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.719957:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.719958:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.719998:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720001:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720003:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720005:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720007:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720011:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720018:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720021:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720024:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720028:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720030:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720036:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720037:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720040:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720042:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720044:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720046:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720047:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720048:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720049:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720051:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720053:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720055:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720058:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720060:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720062:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720065:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720066:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720069:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720070:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720072:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720074:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720076:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720077:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720079:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720081:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720082:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720083:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720085:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720091:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720092:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720095:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720096:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720098:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720099:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720101:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720102:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720103:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720104:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720105:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720106:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720108:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720109:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720110:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720112:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720114:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720126:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720128:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720131:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720132:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720134:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720136:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720138:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720139:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720140:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720143:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720145:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720148:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720150:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720152:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720154:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720158:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720164:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720168:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720170:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720172:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720173:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720175:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720176:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720177:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720178:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720180:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720181:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720183:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720184:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720185:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720188:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720189:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720191:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720191:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720193:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720194:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720195:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720196:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720197:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720198:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720199:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720200:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720202:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720203:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720204:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720206:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720206:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720208:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720208:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720212:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720214:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720228:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720229:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720230:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720232:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720234:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720235:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720237:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720238:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720239:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720242:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720243:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720245:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720246:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720248:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720249:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720250:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720251:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720252:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720254:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720266:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720267:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720268:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720270:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720271:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720272:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720273:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720275:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720275:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720276:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720278:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720279:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720287:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720288:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720291:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720293:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720295:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720297:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720300:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720302:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720306:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720308:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720311:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720313:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720315:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720318:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720320:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720321:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720322:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720324:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720325:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720326:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720328:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720329:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720331:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720333:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720333:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720335:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720336:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720337:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720338:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720339:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720340:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720340:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720342:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720347:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720348:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720349:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720351:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720352:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720355:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720356:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720357:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720358:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720359:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720360:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720361:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720362:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720363:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720364:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720365:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720367:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720368:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720369:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720371:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720373:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720373:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720375:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720375:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720377:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720378:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720379:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720379:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720380:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720381:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720383:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720384:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720385:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720389:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720390:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720393:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720394:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720396:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720396:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720398:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720399:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720400:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720401:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720401:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720403:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720404:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720405:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720406:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720407:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720408:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720410:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720411:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720412:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720413:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720414:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720415:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720416:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720417:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720418:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720419:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720420:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720421:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720422:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720423:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720425:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720426:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720430:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720432:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720432:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720434:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720435:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720436:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720437:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720437:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720439:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720440:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720441:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720442:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720443:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720445:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720447:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720448:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720449:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720450:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720451:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720452:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720453:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720453:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720454:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720456:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720457:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720458:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720459:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720460:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720461:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720463:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720464:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720465:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720466:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720470:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720471:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720472:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720473:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720474:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720475:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720476:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720477:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720478:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720480:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720481:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720484:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720485:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720488:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720489:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720491:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720493:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720494:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720495:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720497:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720499:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720501:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720502:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720505:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720507:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720509:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720512:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720514:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720516:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720517:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720520:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720522:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720524:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720531:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720532:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720536:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720538:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720540:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720542:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720545:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720547:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720552:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720553:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720557:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720558:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720560:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720562:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720563:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720564:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720565:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720567:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720568:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720569:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720571:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720572:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720573:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720575:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720576:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720577:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720578:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720580:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720581:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720582:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720583:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720584:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720585:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720590:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720591:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720593:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720594:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720596:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720598:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720599:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720601:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720602:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720603:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720604:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720605:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720606:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720607:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720608:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720610:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720611:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720612:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720613:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720614:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720616:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720617:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720618:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720619:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720620:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720621:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720622:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720623:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720623:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720625:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720626:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720627:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720628:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720633:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720634:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720637:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720638:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720640:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720641:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720642:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720643:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720644:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720645:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720645:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720647:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720648:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720649:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720650:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720651:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720653:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720654:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720655:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720656:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720657:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720658:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720659:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720660:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720661:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720662:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720663:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720664:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720665:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720666:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720668:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720669:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720670:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720674:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720676:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720677:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720678:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720679:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720680:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720681:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720682:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720683:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720685:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720686:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720687:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720688:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720689:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720692:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720692:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720694:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720694:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720696:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720697:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720698:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720698:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720699:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720700:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720701:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720703:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720704:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720705:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720706:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720708:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720708:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720710:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720710:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720714:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720716:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720717:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720718:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720719:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720720:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720722:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720723:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720724:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720725:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720726:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720729:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720730:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720731:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720732:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720733:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720734:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720735:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720736:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720736:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720738:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720739:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720740:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720741:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720742:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720743:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720745:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720746:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720747:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720748:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720749:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720750:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720751:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720754:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720755:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720757:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720758:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720759:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720760:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720762:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720763:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720765:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720766:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720768:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720768:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720770:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720771:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720772:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720772:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720773:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720774:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720776:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720777:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720778:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720779:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720780:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720782:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720783:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720784:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720785:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720786:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720787:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720788:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720788:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720789:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720790:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720794:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720795:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720797:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720798:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720799:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720802:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720802:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720804:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720805:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720806:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720807:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720808:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720809:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720810:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720811:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720812:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720813:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720814:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720816:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720817:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720818:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720819:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720820:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720821:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720822:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720823:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720824:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720825:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720826:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720827:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720828:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720829:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720831:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720834:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720836:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720838:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720839:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720841:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720841:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720843:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720844:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720845:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720845:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720846:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720848:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720849:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720850:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720851:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720852:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720853:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720855:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720856:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720857:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720858:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720859:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720860:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720861:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720862:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720862:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720864:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720865:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720866:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720867:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720868:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720869:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720871:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720884:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720886:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720887:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720888:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720889:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720890:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720891:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720892:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720893:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720895:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720896:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720897:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720898:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720899:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720902:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720903:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720904:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720905:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720906:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720907:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720908:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720908:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720909:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720911:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720912:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720913:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720914:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720915:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720916:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720918:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720919:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720920:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720921:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720925:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720926:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720927:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720928:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720929:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720931:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720932:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720933:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720934:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720936:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720937:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720939:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720940:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720942:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720942:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720943:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720945:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720946:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720946:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720947:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720948:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720949:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720951:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720952:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720953:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720954:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720956:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720956:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720958:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720958:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.720975:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.720977:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.720978:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.720982:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.720983:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.720985:0:14647:0:(lmv_obd.c:939:lmv_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720987:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720988:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.720989:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.720991:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720992:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.720995:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.720996:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.720998:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.720999:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.721000:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.721002:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.721003:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00800000:00000001:0.0:1773941498.721003:0:14647:0:(lmv_obd.c:902:lmv_iocontrol()) Process entered 00800000:00000080:0.0:1773941498.721004:0:14647:0:(lmv_obd.c:904:lmv_iocontrol()) lustre-clilmv-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00800000:00000001:0.0:1773941498.721006:0:14647:0:(lmv_obd.c:935:lmv_iocontrol()) Process leaving (rc=18446744073709551597 : -19 : ffffffffffffffed) 00000080:00000001:0.0:1773941498.721007:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551597 : -19 : ffffffffffffffed) 00000080:00000001:0.0:1773941498.721008:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551597 : -19 : 0xffffffffffffffed) 00000080:00000010:0.0:1773941498.721009:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.721011:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551597 : -19 : ffffffffffffffed) 00000080:00000001:0.0:1773941498.721012:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551597 : -19 : ffffffffffffffed) 00000080:00000001:0.0:1773941498.721013:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.721014:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.721016:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.721016:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.721017:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.721019:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.721020:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.721022:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.721023:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.721027:0:14647:0:(obd_class.h:1067:obd_statfs()) Process entered 00020000:00000004:0.0:1773941498.721032:0:14647:0:(obd_class.h:1083:obd_statfs()) lustre-OST0000-osc-ffff988d08f0a000: age -780, max_age 220 00000008:00000001:0.0:1773941498.721035:0:14647:0:(osc_request.c:3449:osc_statfs()) Process entered 00000020:00000040:0.0:1773941498.721036:0:14647:0:(genops.c:1229:class_import_get()) import ffff988d180dc800 refcount=3 obd=lustre-OST0000-osc-ffff988d08f0a000 00000100:00000010:0.0:1773941498.721039:0:14647:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10836a00. 00000020:00000040:0.0:1773941498.721041:0:14647:0:(genops.c:1229:class_import_get()) import ffff988d180dc800 refcount=4 obd=lustre-OST0000-osc-ffff988d08f0a000 00000020:00000001:0.0:1773941498.721043:0:14647:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941498.721044:0:14647:0:(genops.c:1242:class_import_put()) import ffff988d180dc800 refcount=3 obd=lustre-OST0000-osc-ffff988d08f0a000 00000020:00000001:0.0:1773941498.721046:0:14647:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941498.721047:0:14647:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941498.721049:0:14647:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941498.721050:0:14647:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941498.721053:0:14647:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff988d04c1a400. 00000100:00000001:0.0:1773941498.721056:0:14647:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.721058:0:14647:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1773941498.721059:0:14647:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941498.721061:0:14647:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d09343d80. 00000100:00000001:0.0:1773941498.721062:0:14647:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330245070208 : -113743464481408 : ffff988d09343d80) 00000100:00000001:0.0:1773941498.721064:0:14647:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941498.721065:0:14647:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367699456 : -113743341852160 : ffff988d10836a00) 00000100:00000001:0.0:1773941498.721067:0:14647:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941498.721069:0:14647:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.721070:0:14647:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941498.721071:0:14647:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941498.721075:0:14647:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10836a00 x1860112291475456/t0(0) o13->lustre-OST0000-osc-ffff988d08f0a000@192.168.204.114@tcp:7/4 lens 224/368 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941498.721080:0:14647:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941498.721081:0:14647:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941498.721083:0:14647:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941498.721084:0:14647:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941498.721086:0:14647:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941498.721088:0:14647:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10836a00 pname:cluuid:pid:xid:nid:opc:job lfs:e9ee4b2c-2bac-44e1-850a-8bf1479379ac:14647:1860112291475456:192.168.204.114@tcp:13:lfs.0 00000100:00000001:0.0:1773941498.721090:0:14647:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941498.721092:0:14647:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941498.721093:0:14647:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941498.721098:0:14647:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941498.721100:0:14647:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d065f8000. 02000000:00000001:0.0:1773941498.721101:0:14647:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941498.721104:0:14647:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941498.721106:0:14647:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f420. 00000400:00000010:0.0:1773941498.721108:0:14647:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833aa0. 00000100:00000200:0.0:1773941498.721111:0:14647:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291475456, portal 4 00000100:00000001:0.0:1773941498.721113:0:14647:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941498.721114:0:14647:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367699456 : -113743341852160 : ffff988d10836a00) 00000100:00000040:0.0:1773941498.721120:0:14647:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10836a00 x1860112291475456/t0(0) o13->lustre-OST0000-osc-ffff988d08f0a000@192.168.204.114@tcp:7/4 lens 224/368 e 0 to 0 dl 1773941514 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941498.721128:0:14647:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941498.721130:0:14647:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941498.721133:0:14647:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833000. 00000100:00000200:0.0:1773941498.721136:0:14647:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 7, xid 1860112291475456, offset 0 00000400:00000200:0.0:1773941498.721141:0:14647:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941498.721146:0:14647:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941498.721151:0:14647:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941498.721154:0:14647:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941498.721157:0:14647:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941498.721160:0:14647:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941498.721163:0:14647:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:0.0:1773941498.721167:0:14647:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [103:103:256:4294967295] 192.168.204.114@tcp LPNI seq info [103:103:8:4294967295] 00000400:00000200:0.0:1773941498.721171:0:14647:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941498.721178:0:14647:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:0.0:1773941498.721181:0:14647:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:0.0:1773941498.721184:0:14647:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d04c1a100. 00000800:00000200:0.0:1773941498.721188:0:14647:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:0.0:1773941498.721192:0:14647:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:0.0:1773941498.721194:0:14647:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d04c1a100 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1773941498.721204:0:14647:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941498.721212:0:14647:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff988d08f0a000: using a timeout of 4291 usecs (5 jiffies) 00000400:00000200:0.0:1773941498.721227:0:14647:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941498.721230:0:14647:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff988d08f0a000: PM QoS new deadline estimation for cpu 0 is 4294889067 00000100:00000001:0.0:1773941498.721236:0:14647:0:(niobuf.c:684:kick_cpu_latency()) lustre-OST0000-osc-ffff988d08f0a000: PM QoS request active for cpu 0, simply extend its deadline from 4294889065 00000100:00000001:0.0:1773941498.721240:0:14647:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff988d08f0a000: PM QoS new deadline estimation for cpu 1 is 4294889067 00000100:00000001:0.0:1773941498.721244:0:14647:0:(niobuf.c:684:kick_cpu_latency()) lustre-OST0000-osc-ffff988d08f0a000: PM QoS request active for cpu 1, simply extend its deadline from 4294889065 00000100:00000001:0.0:1773941498.721248:0:14647:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941498.721251:0:14647:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.721253:0:14647:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941498.721255:0:14647:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1773941498.721257:0:14647:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d09343d80 going to sleep for 11 seconds 00000100:00000001:0.0:1773941498.721260:0:14647:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941498.721261:0:14647:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941498.721303:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941498.721307:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d04c1a100. 00000400:00000200:1.0:1773941498.721311:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941498.721316:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941498.721320:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833000 00000400:00000010:1.0:1773941498.721330:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833000. 00000100:00000001:1.0:1773941498.721333:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941498.721339:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10836a00 x1860112291475456/t0(0) o13->lustre-OST0000-osc-ffff988d08f0a000@192.168.204.114@tcp:7/4 lens 224/368 e 0 to 0 dl 1773941514 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.721348:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941498.721351:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d10836a00 x1860112291475456/t0(0) o13->lustre-OST0000-osc-ffff988d08f0a000@192.168.204.114@tcp:7/4 lens 224/368 e 0 to 0 dl 1773941514 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.721358:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.721360:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:1.0:1773941498.722361:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941498.722367:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941498.722370:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941498.722371:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941498.722374:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.722377:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941498.722381:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.722384:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.722386:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.722389:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 368 into portal 4 MB=0x69bc324603000 00000400:00000200:1.0:1773941498.722391:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.722394:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.204.114@tcp of length 368/368 into md 0x859 [1] + 224 00000800:00000001:1.0:1773941498.722397:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941498.722402:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941498.722404:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941498.722406:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:1.0:1773941498.722409:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941498.722412:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10836a00 x1860112291475456/t0(0) o13->lustre-OST0000-osc-ffff988d08f0a000@192.168.204.114@tcp:7/4 lens 224/368 e 0 to 0 dl 1773941514 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941498.722418:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=368 offset=224 replen=368 req@ffff988d10836a00 x1860112291475456/t0(0) o13->lustre-OST0000-osc-ffff988d08f0a000@192.168.204.114@tcp:7/4 lens 224/368 e 0 to 0 dl 1773941514 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.722430:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1773941498.722439:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941498.722441:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941498.722459:0:14647:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941498.722464:0:14647:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941498.722466:0:14647:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941498.722468:0:14647:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f420. 00000400:00000200:0.0:1773941498.722472:0:14647:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833aa0 00000400:00000010:0.0:1773941498.722474:0:14647:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833aa0. 00000100:00000001:0.0:1773941498.722477:0:14647:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941498.722483:0:14647:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10836a00 x1860112291475456/t0(0) o13->lustre-OST0000-osc-ffff988d08f0a000@192.168.204.114@tcp:7/4 lens 224/368 e 0 to 0 dl 1773941514 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941498.722490:0:14647:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10836a00 x1860112291475456/t0(0) o13->lustre-OST0000-osc-ffff988d08f0a000@192.168.204.114@tcp:7/4 lens 224/368 e 0 to 0 dl 1773941514 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941498.722495:0:14647:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941498.722496:0:14647:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941498.722498:0:14647:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941498.722500:0:14647:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941498.722501:0:14647:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941498.722503:0:14647:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941498.722505:0:14647:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941498.722510:0:14647:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d180dcca0 time=1773941498 v=5 (0 0 0 0) 00000100:00001000:0.0:1773941498.722513:0:14647:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d180dcc10 time=1 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941498.722523:0:14647:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941498.722525:0:14647:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941498.722527:0:14647:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1773941498.722532:0:14647:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff988d10836a00 x1860112291475456/t0(0) o13->lustre-OST0000-osc-ffff988d08f0a000@192.168.204.114@tcp:7/4 lens 224/368 e 0 to 0 dl 1773941514 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1773941498.722539:0:14647:0:(ldlm_request.c:1535:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.722542:0:14647:0:(client.c:3065:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1773941498.722544:0:14647:0:(client.c:3077:ptlrpc_free_committed()) lustre-OST0000-osc-ffff988d08f0a000: skip recheck: last_committed 0 00000100:00000001:0.0:1773941498.722545:0:14647:0:(client.c:3078:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1773941498.722547:0:14647:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941498.722551:0:14647:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10836a00 x1860112291475456/t0(0) o13->lustre-OST0000-osc-ffff988d08f0a000@192.168.204.114@tcp:7/4 lens 224/368 e 0 to 0 dl 1773941514 ref 2 fl Rpc:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941498.722559:0:14647:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941498.722561:0:14647:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941498.722562:0:14647:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1773941498.722565:0:14647:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10836a00 x1860112291475456/t0(0) o13->lustre-OST0000-osc-ffff988d08f0a000@192.168.204.114@tcp:7/4 lens 224/368 e 0 to 0 dl 1773941514 ref 2 fl Interpret:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941498.722571:0:14647:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10836a00 pname:cluuid:pid:xid:nid:opc:job lfs:e9ee4b2c-2bac-44e1-850a-8bf1479379ac:14647:1860112291475456:192.168.204.114@tcp:13:lfs.0 00000100:00000001:0.0:1773941498.722574:0:14647:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941498.722575:0:14647:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.722577:0:14647:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1773941498.722578:0:14647:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941498.722580:0:14647:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10836a00 x1860112291475456/t0(0) o13->lustre-OST0000-osc-ffff988d08f0a000@192.168.204.114@tcp:7/4 lens 224/368 e 0 to 0 dl 1773941514 ref 2 fl Complete:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941498.722584:0:14647:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1773941498.722586:0:14647:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d09343d80. 00000100:00000001:0.0:1773941498.722588:0:14647:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1773941498.722589:0:14647:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1773941498.722592:0:14647:0:(osc_request.c:3497:osc_statfs()) Process leaving 00000100:00000001:0.0:1773941498.722593:0:14647:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941498.722595:0:14647:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10836a00 x1860112291475456/t0(0) o13->lustre-OST0000-osc-ffff988d08f0a000@192.168.204.114@tcp:7/4 lens 224/368 e 0 to 0 dl 1773941514 ref 1 fl Complete:RQU/200/0 rc 0/0 job:'lfs.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941498.722608:0:14647:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1773941498.722609:0:14647:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1773941498.722611:0:14647:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d065f8000. 02000000:00000001:0.0:1773941498.722613:0:14647:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1773941498.722614:0:14647:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941498.722615:0:14647:0:(genops.c:1242:class_import_put()) import ffff988d180dc800 refcount=2 obd=lustre-OST0000-osc-ffff988d08f0a000 00000020:00000001:0.0:1773941498.722616:0:14647:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1773941498.722618:0:14647:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff988d04c1a400. 02000000:00000001:0.0:1773941498.722619:0:14647:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1773941498.722620:0:14647:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1773941498.722621:0:14647:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10836a00. 00000100:00000001:0.0:1773941498.722623:0:14647:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941498.722624:0:14647:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00020000:00000004:0.0:1773941498.722626:0:14647:0:(obd_class.h:1109:obd_statfs()) lustre-OST0000-osc-ffff988d08f0a000: update ffff988d06460450 cache blocks 368/374 objects 11840/12362 00020000:00000001:0.0:1773941498.722628:0:14647:0:(obd_class.h:1128:obd_statfs()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:0.0:1773941498.722631:0:14647:0:(lov_obd.c:1128:lov_iocontrol()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.722632:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1773941498.722634:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.722635:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.722636:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.722660:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.722662:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.722665:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.722667:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.722669:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.722672:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.722673:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.722675:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.722677:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.722680:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722681:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722683:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.722684:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.722686:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722692:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722695:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.722696:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.722698:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.722699:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.722701:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.722702:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.722703:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.722704:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.722705:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.722707:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722708:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722710:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.722711:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.722713:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722714:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722716:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.722717:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.722719:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.722719:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.722721:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.722723:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.722724:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.722725:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.722725:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.722727:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722729:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722731:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.722732:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.722734:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722735:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722736:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.722738:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.722743:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.722744:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.722747:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.722750:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.722752:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.722753:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.722755:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.722759:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722762:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722764:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.722767:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.722771:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722774:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722779:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.722781:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.722784:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.722785:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.722788:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.722791:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.722792:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.722794:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.722795:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.722798:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722800:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722802:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.722805:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.722807:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722810:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722813:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.722815:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.722819:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.722820:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.722824:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.722833:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.722835:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.722837:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.722838:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.722840:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722842:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722844:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.722845:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.722847:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722849:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722852:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.722853:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.722855:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.722856:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.722859:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.722860:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.722862:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.722863:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.722864:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.722866:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722867:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722868:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.722869:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.722871:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722872:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722874:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.722875:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.722876:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.722877:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.722878:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.722879:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.722880:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.722881:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.722884:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.722886:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722888:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722889:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.722890:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.722892:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722893:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722895:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.722896:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.722898:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.722898:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.722900:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.722901:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.722902:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.722902:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.722903:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.722904:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722906:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722907:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.722908:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.722909:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722910:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722912:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.722913:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.722914:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.722915:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.722916:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.722917:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.722918:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.722918:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.722919:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.722921:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722922:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722926:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.722927:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.722928:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722930:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722932:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.722933:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.722935:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.722936:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.722937:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.722938:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.722939:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.722940:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.722941:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.722942:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722943:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722944:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.722945:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.722947:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722948:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722949:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.722950:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.722952:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.722952:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.722953:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.722954:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.722955:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.722956:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.722957:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.722958:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722976:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722978:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.722979:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.722984:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722985:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.722989:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.722991:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.722994:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.722995:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.722996:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.722998:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.722999:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723001:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723002:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723004:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723006:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723007:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723009:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723011:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723012:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723014:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723016:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723017:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723018:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723020:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723021:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723022:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723023:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723024:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723025:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723027:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723028:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723029:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723031:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723032:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723034:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723039:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723041:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723042:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723043:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723044:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723046:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723046:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723047:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723049:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723050:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723051:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723052:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723053:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723054:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723057:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723058:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723059:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723060:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723061:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723062:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723063:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723063:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723064:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723066:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723067:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723068:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723069:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723070:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723071:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723073:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723074:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723075:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723076:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723080:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723081:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723083:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723084:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723084:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723086:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723087:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723088:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723090:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723091:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723092:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723094:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723095:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723097:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723097:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723099:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723100:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723101:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723101:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723102:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723103:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723104:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723106:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723107:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723108:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723109:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723111:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723112:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723114:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723115:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723117:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723119:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723120:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723126:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723127:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723130:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723132:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723134:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723137:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723139:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723142:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723146:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723148:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723151:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723153:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723155:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723158:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723160:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723161:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723162:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723164:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723165:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723167:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723168:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723170:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723171:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723173:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723174:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723175:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723176:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723177:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723179:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723180:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723180:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723181:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723183:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723188:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723190:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723191:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723193:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723194:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723197:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723198:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723199:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723200:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723201:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723202:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723203:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723204:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723205:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723206:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723207:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723208:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723210:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723211:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723212:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723214:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723214:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723230:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723231:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723233:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723235:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723236:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723237:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723238:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723240:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723241:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723242:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723243:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723249:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723250:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723253:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723254:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723256:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723257:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723258:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723260:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723261:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723262:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723263:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723264:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723265:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723266:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723268:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723269:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723270:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723272:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723272:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723274:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723274:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723276:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723277:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723278:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723278:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723279:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723280:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723282:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723283:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723284:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723285:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723286:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723288:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723291:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723293:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723294:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723296:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723297:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723298:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723299:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723300:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723301:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723302:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723303:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723304:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723306:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723307:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723309:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723310:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723311:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723312:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723313:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723314:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723315:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723316:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723317:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723318:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723319:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723320:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723321:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723322:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723324:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723325:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723326:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723327:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723328:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723332:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723333:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723335:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723335:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723336:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723338:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723339:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723340:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723342:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723343:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723344:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723347:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723347:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723349:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723350:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723351:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723352:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723353:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723354:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723354:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723356:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723357:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723358:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723359:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723360:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723361:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723363:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723364:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723365:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723366:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723367:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723368:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723369:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723374:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723375:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723376:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723377:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723379:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723380:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723381:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723382:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723385:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723386:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723387:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723388:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723389:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723390:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723391:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723392:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723393:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723394:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723395:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723396:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723397:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723399:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723400:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723401:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723402:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723403:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723404:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723405:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723406:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723407:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723408:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723409:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723410:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723414:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723415:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723417:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723419:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723420:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723422:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723423:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723425:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723426:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723427:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723428:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723429:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723430:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723430:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723432:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723433:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723434:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723435:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723436:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723438:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723439:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723440:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723441:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723442:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723443:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723444:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723445:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723446:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723447:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723448:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723449:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723450:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723451:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723455:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723456:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723459:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723460:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723462:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723463:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723464:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723465:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723466:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723467:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723468:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723469:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723470:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723471:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723473:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723474:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723475:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723476:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723477:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723479:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723479:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723480:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723482:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723483:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723483:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723484:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723485:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723486:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723487:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723489:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723490:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723491:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723493:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723496:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723499:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723500:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723503:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723505:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723507:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723508:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723510:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723512:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723514:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723516:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723519:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723521:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723523:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723528:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723529:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723532:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723533:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723535:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723538:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723539:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723540:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723542:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723545:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723547:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723549:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723552:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723565:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723566:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723569:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723570:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723572:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723573:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723579:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723580:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723582:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723583:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723584:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723586:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723587:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723589:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723590:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723591:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723593:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723599:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723600:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723602:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723602:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723604:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723605:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723606:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723607:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723608:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723609:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723611:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723612:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723614:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723615:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723616:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723618:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723619:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723620:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723621:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723622:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723623:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723624:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723628:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723629:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723630:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723632:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723633:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723634:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723636:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723637:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723639:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723640:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723642:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723642:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723644:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723645:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723646:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723646:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723647:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723649:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723650:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723651:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723652:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723653:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723654:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723656:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723657:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723658:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723659:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723660:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723661:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723662:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723663:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723663:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723665:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723669:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723670:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723672:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723673:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723675:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723677:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723678:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723679:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723680:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723681:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723683:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723683:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723684:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723685:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723686:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723687:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723689:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723690:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723691:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723692:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723694:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723694:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723696:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723696:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723698:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723699:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723700:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723700:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723701:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723702:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723704:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723705:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723706:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723711:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723712:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723714:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723715:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723717:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723718:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723719:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723720:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723721:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723722:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723723:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723724:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723725:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723726:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723727:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723729:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723730:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723731:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723732:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723733:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723734:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723735:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723736:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723737:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723738:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723739:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723740:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723741:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723742:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723743:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723745:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723746:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723747:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723751:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723753:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723754:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723755:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723756:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723757:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723758:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723759:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723760:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723761:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723762:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723763:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723765:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723766:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723768:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723769:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723770:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723771:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723772:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723773:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723774:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723775:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723776:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723777:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723778:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723779:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723780:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723781:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723783:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723784:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723785:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723786:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723787:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723791:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723792:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723793:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723794:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723795:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723796:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723797:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723798:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723800:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723801:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723802:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723804:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723805:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723807:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723808:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723809:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723810:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723811:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723811:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723812:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723813:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723815:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723816:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723817:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723818:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723819:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723821:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723821:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723823:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723823:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723825:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723826:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723827:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723830:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723831:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723832:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723834:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723835:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723836:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723837:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723838:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723841:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723842:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723843:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723844:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723845:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723846:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723847:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723848:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723849:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723850:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723851:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723852:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723853:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723855:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723856:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723857:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723858:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723859:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723860:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723861:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723862:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723863:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723864:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723865:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723866:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723870:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723871:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723872:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723873:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723875:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723877:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723878:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723879:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723880:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723881:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723883:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723884:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723884:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723885:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723886:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723888:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723889:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723890:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723891:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723892:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723894:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723895:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723896:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723897:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723898:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723899:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723900:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723900:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723901:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723902:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723904:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723905:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723906:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723910:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723911:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723913:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723914:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723916:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723916:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723918:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723919:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723920:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723921:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723921:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723923:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723924:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723925:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723926:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723927:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723929:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723930:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723931:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723932:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723933:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723934:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723935:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723936:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723937:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723938:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723939:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723940:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723941:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723942:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723943:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723945:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723946:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723950:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723951:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723952:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723953:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723955:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.723956:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.723956:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.723957:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.723959:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723984:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723985:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.723987:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.723988:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723989:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.723992:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.723993:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.723995:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.723996:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.723997:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.723999:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.724000:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.724000:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.724001:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.724003:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.724004:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.724005:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.724007:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.724008:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.724009:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.724010:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.724011:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.724013:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.724013:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.724019:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.724020:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.724022:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.724023:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.724024:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.724025:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.724027:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.724028:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.724029:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.724030:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.724031:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.724034:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.724035:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.724036:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.724037:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.724038:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.724039:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.724040:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.724041:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.724041:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.724043:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.724044:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.724045:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.724046:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.724047:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.724049:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.724050:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.724051:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.724053:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.724053:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.724055:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.724056:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.724057:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.724061:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.724062:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.724064:0:14647:0:(lov_obd.c:988:lov_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.724065:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.724066:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551605 : -11 : 0xfffffffffffffff5) 00000080:00000010:0.0:1773941498.724067:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.724069:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.724070:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000080:00000001:0.0:1773941498.724072:0:14647:0:(dir.c:1982:ll_dir_ioctl()) Process entered 00000080:00200080:0.0:1773941498.724073:0:14647:0:(dir.c:1984:ll_dir_ioctl()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) cmd=c00866a4 arg=7ffdd0765da0 00000080:00000001:0.0:1773941498.724075:0:14647:0:(llite_lib.c:3403:ll_iocontrol()) Process entered 00000020:00000001:0.0:1773941498.724075:0:14647:0:(class_obd.c:240:obd_ioctl_getdata()) Process entered 00000020:00000010:0.0:1773941498.724077:0:14647:0:(class_obd.c:267:obd_ioctl_getdata()) kmalloced '(data)': 592 at ffff988d065f8c00. 00000020:00000001:0.0:1773941498.724078:0:14647:0:(class_obd.c:302:obd_ioctl_getdata()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.724079:0:14647:0:(obd_class.h:1193:obd_iocontrol()) Process entered 00020000:00000001:0.0:1773941498.724079:0:14647:0:(lov_obd.c:956:lov_iocontrol()) Process entered 00020000:00000080:0.0:1773941498.724080:0:14647:0:(lov_obd.c:958:lov_iocontrol()) lustre-clilov-ffff988d08f0a000: cmd=c00866a4 len=592 karg=ffff988d065f8c00 uarg=0000000000000000 00020000:00000001:0.0:1773941498.724082:0:14647:0:(lov_obd.c:984:lov_iocontrol()) Process leaving (rc=18446744073709551597 : -19 : ffffffffffffffed) 00000080:00000001:0.0:1773941498.724083:0:14647:0:(obd_class.h:1208:obd_iocontrol()) Process leaving (rc=18446744073709551597 : -19 : ffffffffffffffed) 00000080:00000001:0.0:1773941498.724084:0:14647:0:(llite_lib.c:3838:ll_obd_statfs()) Process leaving via out_statfs (rc=18446744073709551597 : -19 : 0xffffffffffffffed) 00000080:00000010:0.0:1773941498.724085:0:14647:0:(llite_lib.c:3840:ll_obd_statfs()) kvfree 'data': 592 at ffff988d065f8c00. 00000080:00000001:0.0:1773941498.724086:0:14647:0:(llite_lib.c:3441:ll_iocontrol()) Process leaving (rc=18446744073709551597 : -19 : ffffffffffffffed) 00000080:00000001:0.0:1773941498.724087:0:14647:0:(dir.c:2910:ll_dir_ioctl()) Process leaving (rc=18446744073709551597 : -19 : ffffffffffffffed) 00000080:00000001:0.0:1773941498.724093:0:14647:0:(dir.c:2980:ll_dir_release()) Process entered 00000080:00000001:0.0:1773941498.724095:0:14647:0:(file.c:420:ll_file_release()) Process entered 00000020:00000001:0.0:1773941498.724097:0:14647:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffb6ec819e7db0 00000080:00200008:0.0:1773941498.724098:0:14647:0:(file.c:424:ll_file_release()) START file /:[0x200000007:0x1:0x0](ffff988d20529988), flags 100000 00000080:00400000:0.0:1773941498.724101:0:14647:0:(statahead.c:1714:ll_deauthorize_statahead()) deauthorize statahead for [0x200000007:0x1:0x0] 00000080:00000010:0.0:1773941498.724104:0:14647:0:(file.c:80:ll_file_data_put()) slab-freed '(lfd)': 288 at ffff988d0a346140. 00000080:00000001:0.0:1773941498.724106:0:14647:0:(file.c:439:ll_file_release()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1773941498.724107:0:14647:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffb6ec819e7db0 00000080:00000008:0.0:1773941498.724109:0:14647:0:(file.c:464:ll_file_release()) COMPLETED file /:[0x200000007:0x1:0x0](ffff988d20529988), flags 100000, rc = 0 00000080:00000001:0.0:1773941498.724110:0:14647:0:(file.c:466:ll_file_release()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.724116:0:14647:0:(dir.c:2981:ll_dir_release()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941498.727003:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294889067, at 4294889068 00000100:00000010:0.0:1773941498.727011:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580360. 00000100:00000040:1.0:1773941498.728649:0:804:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294889067, at 4294889069 00000100:00000010:1.0:1773941498.728656:0:804:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580240. 00000020:00000001:1.0:1773941498.735259:0:14663:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffb6ec81cafd08 00000080:00200008:1.0:1773941498.735265:0:14663:0:(file.c:6291:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff988d20529988), request_mask 2047, flags 0, foreign 0 00000080:00000001:1.0:1773941498.735271:0:14663:0:(file.c:6184:ll_inode_revalidate()) Process entered 00000020:00000001:1.0:1773941498.735273:0:14663:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffb6ec81cafc80 00000080:00200000:1.0:1773941498.735276:0:14663:0:(file.c:6186:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988),name=/ 00000080:00000010:1.0:1773941498.735280:0:14663:0:(llite_lib.c:3907:ll_prep_md_op_data()) kmalloced '(op_data)': 384 at ffff988d08a4e400. 00800000:00000002:1.0:1773941498.735286:0:14663:0:(lmv_obd.c:3847:lmv_stripe_object_get()) get ffff988d10872600 2 00000080:00000001:1.0:1773941498.735289:0:14663:0:(namei.c:943:ll_intent_lock()) Process entered 00000020:00000010:1.0:1773941498.735294:0:14663:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 640 at ffff988d1118bc00. 00800000:00000001:1.0:1773941498.735297:0:14663:0:(lmv_intent.c:556:lmv_intent_lock()) Process entered 00000020:00000001:1.0:1773941498.735299:0:14663:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffb6ec81cafba0 00800000:00000002:1.0:1773941498.735302:0:14663:0:(lmv_intent.c:563:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:1.0:1773941498.735305:0:14663:0:(lmv_intent.c:437:lmv_intent_lookup()) Process entered 00800000:00000001:1.0:1773941498.735307:0:14663:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:1.0:1773941498.735309:0:14663:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:1.0:1773941498.735311:0:14663:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:1.0:1773941498.735313:0:14663:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:1.0:1773941498.735315:0:14663:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:1.0:1773941498.735317:0:14663:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:1.0:1773941498.735319:0:14663:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:1.0:1773941498.735322:0:14663:0:(lmv_intent.c:490:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000020:00000010:1.0:1773941498.735326:0:14663:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 640 at ffff988d1118a000. 00000002:00000001:1.0:1773941498.735329:0:14663:0:(mdc_locks.c:1500:mdc_intent_lock()) Process entered 00000020:00000001:1.0:1773941498.735330:0:14663:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffb6ec81cafa70 00000002:00010000:1.0:1773941498.735332:0:14663:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:1.0:1773941498.735335:0:14663:0:(mdc_locks.c:1404:mdc_revalidate_lock()) Process entered 00000002:00000001:1.0:1773941498.735336:0:14663:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:1.0:1773941498.735339:0:14663:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:1.0:1773941498.735344:0:14663:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff988d21ec1800 count: 2 00010000:00000001:1.0:1773941498.735347:0:14663:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941498.735355:0:14663:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000020:00000010:1.0:1773941498.735359:0:14663:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 128 at ffff988d0716b480. 00010000:00000001:1.0:1773941498.735362:0:14663:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:1.0:1773941498.735366:0:14663:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff988d08f0a000 lock: ffff988d00030000/0x8e86767622e28c6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd03f0241a6d949fd expref: -99 pid: 14647 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 02000000:00000001:1.0:1773941498.735375:0:14663:0:(sec.c:1015:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:1.0:1773941498.735380:0:14663:0:(sec.c:1031:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941498.735382:0:14663:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff988d08f0a000 lock: ffff988d00030000/0x8e86767622e28c6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd03f0241a6d949fd expref: -99 pid: 14647 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00000002:00000001:1.0:1773941498.735388:0:14663:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:1.0:1773941498.735390:0:14663:0:(mdc_locks.c:1456:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:1.0:1773941498.735391:0:14663:0:(mdc_locks.c:1523:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:1.0:1773941498.735393:0:14663:0:(lmv_intent.c:511:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:1.0:1773941498.735395:0:14663:0:(lmv_intent.c:596:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:1.0:1773941498.735398:0:14663:0:(namei.c:951:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00000001:1.0:1773941498.735401:0:14663:0:(namei.c:989:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:1.0:1773941498.735405:0:14663:0:(lmv_obd.c:3892:lmv_stripe_object_put()) put ffff988d10872600 2 00000080:00000010:1.0:1773941498.735408:0:14663:0:(llite_lib.c:4048:ll_finish_md_op_data()) kfreed 'op_data': 384 at ffff988d08a4e400. 00000080:00000001:1.0:1773941498.735413:0:14663:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:1.0:1773941498.735415:0:14663:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:1.0:1773941498.735418:0:14663:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff988d20529988) 00000080:00010000:1.0:1773941498.735423:0:14663:0:(llite_internal.h:1998:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff988d20529988) for lock 0x8e86767622e28c6 00800000:00000001:1.0:1773941498.735427:0:14663:0:(lmv_obd.c:3932:lmv_set_lock_data()) Process entered 00000002:00000001:1.0:1773941498.735430:0:14663:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:1.0:1773941498.735433:0:14663:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941498.735435:0:14663:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941498.735439:0:14663:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030000 refcount=3 00000020:00000001:1.0:1773941498.735442:0:14663:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:1.0:1773941498.735445:0:14663:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:1.0:1773941498.735448:0:14663:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941498.735450:0:14663:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:1.0:1773941498.735457:0:14663:0:(mdc_locks.c:122:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:1.0:1773941498.735460:0:14663:0:(lmv_obd.c:3937:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:1.0:1773941498.735463:0:14663:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x8e86767622e28c6 from it ffffb6ec81cafcb8 00010000:00000001:1.0:1773941498.735466:0:14663:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941498.735468:0:14663:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941498.735470:0:14663:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030000 refcount=3 00000020:00000001:1.0:1773941498.735472:0:14663:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:1.0:1773941498.735474:0:14663:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:1.0:1773941498.735475:0:14663:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941498.735478:0:14663:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff988d08f0a000 lock: ffff988d00030000/0x8e86767622e28c6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd03f0241a6d949fd expref: -99 pid: 14647 timeout: 0 lvb_type: 0 lru_score: 3 lru_type: 1 00010000:00000001:1.0:1773941498.735484:0:14663:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941498.735485:0:14663:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941498.735486:0:14663:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:1.0:1773941498.735488:0:14663:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:1.0:1773941498.735490:0:14663:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff988d08f0a000 lock: ffff988d00030000/0x8e86767622e28c6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd03f0241a6d949fd expref: -99 pid: 14647 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00010000:00000001:1.0:1773941498.735496:0:14663:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1773941498.735499:0:14663:0:(ldlm_request.c:2131:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1773941498.735500:0:14663:0:(ldlm_request.c:1927:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1773941498.735503:0:14663:0:(ldlm_request.c:2099:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941498.735506:0:14663:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1773941498.735507:0:14663:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941498.735508:0:14663:0:(ldlm_request.c:2141:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941498.735511:0:14663:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 448 at ffff988d08a4e400. 00010000:00000001:1.0:1773941498.735513:0:14663:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941498.735515:0:14663:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941498.735516:0:14663:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941498.735517:0:14663:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:1.0:1773941498.735519:0:14663:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1773941498.735521:0:14663:0:(client.c:2844:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:1.0:1773941498.735524:0:14663:0:(file.c:6244:ll_merge_md_attr()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941498.735529:0:14663:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 1856 at ffff988d18b19800. 00000020:00000001:1.0:1773941498.735535:0:14663:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffb6ec81cafd08 00000080:00000008:1.0:1773941498.735537:0:14663:0:(file.c:6469:ll_getattr_dentry()) COMPLETED file /:[0x200000007:0x1:0x0](ffff988d20529988), request_mask 2047, flags 0, foreign 0 00000020:00000001:1.0:1773941498.735655:0:14663:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffb6ec81cafd08 00000080:00200008:1.0:1773941498.735658:0:14663:0:(file.c:6291:ll_getattr_dentry()) START file /:[0x200000007:0x1:0x0](ffff988d20529988), request_mask 2047, flags 0, foreign 0 00000080:00000001:1.0:1773941498.735661:0:14663:0:(file.c:6184:ll_inode_revalidate()) Process entered 00000020:00000001:1.0:1773941498.735663:0:14663:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffb6ec81cafc80 00000080:00200000:1.0:1773941498.735664:0:14663:0:(file.c:6186:ll_inode_revalidate()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988),name=/ 00000080:00000010:1.0:1773941498.735668:0:14663:0:(llite_lib.c:3907:ll_prep_md_op_data()) kmalloced '(op_data)': 384 at ffff988d08a4e200. 00800000:00000002:1.0:1773941498.735671:0:14663:0:(lmv_obd.c:3847:lmv_stripe_object_get()) get ffff988d10872600 2 00000080:00000001:1.0:1773941498.735673:0:14663:0:(namei.c:943:ll_intent_lock()) Process entered 00800000:00000001:1.0:1773941498.735675:0:14663:0:(lmv_intent.c:556:lmv_intent_lock()) Process entered 00000020:00000001:1.0:1773941498.735676:0:14663:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffb6ec81cafba0 00800000:00000002:1.0:1773941498.735677:0:14663:0:(lmv_intent.c:563:lmv_intent_lock()) INTENT LOCK 'getattr' for [0x200000007:0x1:0x0] '' on [0x200000007:0x1:0x0] 00800000:00000001:1.0:1773941498.735679:0:14663:0:(lmv_intent.c:437:lmv_intent_lookup()) Process entered 00800000:00000001:1.0:1773941498.735680:0:14663:0:(lmv_fld.c:38:lmv_fld_lookup()) Process entered 80000000:00000001:1.0:1773941498.735681:0:14663:0:(fld_request.c:427:fld_client_lookup()) Process entered 80000000:00000001:1.0:1773941498.735682:0:14663:0:(fld_cache.c:479:fld_cache_lookup()) Process entered 80000000:00000001:1.0:1773941498.735683:0:14663:0:(fld_cache.c:498:fld_cache_lookup()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:1.0:1773941498.735685:0:14663:0:(fld_request.c:434:fld_client_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:1.0:1773941498.735686:0:14663:0:(lmv_fld.c:57:lmv_fld_lookup()) FLD lookup got mds #0 for fid=[0x200000007:0x1:0x0] 00800000:00000001:1.0:1773941498.735687:0:14663:0:(lmv_fld.c:65:lmv_fld_lookup()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:1.0:1773941498.735689:0:14663:0:(lmv_intent.c:490:lmv_intent_lookup()) LOOKUP_INTENT with fid1=[0x200000007:0x1:0x0], fid2=[0x200000007:0x1:0x0], name='' -> mds #0 00000002:00000001:1.0:1773941498.735691:0:14663:0:(mdc_locks.c:1500:mdc_intent_lock()) Process entered 00000020:00000001:1.0:1773941498.735692:0:14663:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffb6ec81cafa70 00000002:00010000:1.0:1773941498.735693:0:14663:0:(mdc_locks.c:1506:mdc_intent_lock()) (name: ,[0x200000007:0x1:0x0]) in obj [0x200000007:0x1:0x0], intent: getattr flags 0 00000002:00000001:1.0:1773941498.735695:0:14663:0:(mdc_locks.c:1404:mdc_revalidate_lock()) Process entered 00000002:00000001:1.0:1773941498.735696:0:14663:0:(mdc_locks.c:135:mdc_lock_match()) Process entered 00010000:00000001:1.0:1773941498.735698:0:14663:0:(ldlm_lock.c:1446:ldlm_lock_match_with_skip()) Process entered 00010000:00000040:1.0:1773941498.735700:0:14663:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff988d21ec1800 count: 2 00010000:00000001:1.0:1773941498.735702:0:14663:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941498.735703:0:14663:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941498.735705:0:14663:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:1.0:1773941498.735708:0:14663:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PR) ns: lustre-MDT0000-mdc-ffff988d08f0a000 lock: ffff988d00030000/0x8e86767622e28c6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 3 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd03f0241a6d949fd expref: -99 pid: 14647 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 02000000:00000001:1.0:1773941498.735722:0:14663:0:(sec.c:1015:sptlrpc_import_check_ctx()) Process entered 02000000:00000001:1.0:1773941498.735724:0:14663:0:(sec.c:1031:sptlrpc_import_check_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941498.735726:0:14663:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (0 0) ns: lustre-MDT0000-mdc-ffff988d08f0a000 lock: ffff988d00030000/0x8e86767622e28c6 lrc: 2/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd03f0241a6d949fd expref: -99 pid: 14647 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00000002:00000001:1.0:1773941498.735731:0:14663:0:(mdc_locks.c:141:mdc_lock_match()) Process leaving (rc=4 : 4 : 4) 00000002:00000001:1.0:1773941498.735732:0:14663:0:(mdc_locks.c:1456:mdc_revalidate_lock()) Process leaving (rc=1 : 1 : 1) 00000002:00000001:1.0:1773941498.735733:0:14663:0:(mdc_locks.c:1523:mdc_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:1.0:1773941498.735734:0:14663:0:(lmv_intent.c:511:lmv_intent_lookup()) Process leaving (rc=1 : 1 : 1) 00800000:00000001:1.0:1773941498.735736:0:14663:0:(lmv_intent.c:596:lmv_intent_lock()) Process leaving (rc=1 : 1 : 1) 00000080:00200000:1.0:1773941498.735737:0:14663:0:(namei.c:951:ll_intent_lock()) intent lock 8 on i1 [0x200000007:0x1:0x0] suppgids 0 0: rc 1 00000080:00000001:1.0:1773941498.735739:0:14663:0:(namei.c:989:ll_intent_lock()) Process leaving (rc=1 : 1 : 1) 00800000:00000002:1.0:1773941498.735740:0:14663:0:(lmv_obd.c:3892:lmv_stripe_object_put()) put ffff988d10872600 2 00000080:00000010:1.0:1773941498.735742:0:14663:0:(llite_lib.c:4048:ll_finish_md_op_data()) kfreed 'op_data': 384 at ffff988d08a4e200. 00000080:00000001:1.0:1773941498.735744:0:14663:0:(dcache.c:220:ll_revalidate_it_finish()) Process entered 00000080:00000001:1.0:1773941498.735745:0:14663:0:(dcache.c:223:ll_revalidate_it_finish()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:1.0:1773941498.735747:0:14663:0:(dcache.c:255:ll_lookup_finish_locks()) setting l_data to inode [0x200000007:0x1:0x0](ffff988d20529988) 00000080:00010000:1.0:1773941498.735749:0:14663:0:(llite_internal.h:1998:ll_set_lock_data()) setting l_data to inode [0x200000007:0x1:0x0](ffff988d20529988) for lock 0x8e86767622e28c6 00800000:00000001:1.0:1773941498.735751:0:14663:0:(lmv_obd.c:3932:lmv_set_lock_data()) Process entered 00000002:00000001:1.0:1773941498.735752:0:14663:0:(mdc_locks.c:94:mdc_set_lock_data()) Process entered 00010000:00000001:1.0:1773941498.735753:0:14663:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941498.735754:0:14663:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941498.735755:0:14663:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030000 refcount=3 00000020:00000001:1.0:1773941498.735756:0:14663:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:1.0:1773941498.735758:0:14663:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:1.0:1773941498.735760:0:14663:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941498.735761:0:14663:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000002:00000001:1.0:1773941498.735762:0:14663:0:(mdc_locks.c:122:mdc_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:1.0:1773941498.735763:0:14663:0:(lmv_obd.c:3937:lmv_set_lock_data()) Process leaving (rc=0 : 0 : 0) 00000080:00010000:1.0:1773941498.735764:0:14663:0:(dcache.c:152:ll_intent_drop_lock()) releasing lock with cookie 0x8e86767622e28c6 from it ffffb6ec81cafcb8 00010000:00000001:1.0:1773941498.735766:0:14663:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941498.735767:0:14663:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941498.735773:0:14663:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030000 refcount=3 00000020:00000001:1.0:1773941498.735774:0:14663:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:1.0:1773941498.735776:0:14663:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:1.0:1773941498.735778:0:14663:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941498.735780:0:14663:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PR) ns: lustre-MDT0000-mdc-ffff988d08f0a000 lock: ffff988d00030000/0x8e86767622e28c6 lrc: 3/1,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x10000000000000 nid: local remote: 0xd03f0241a6d949fd expref: -99 pid: 14647 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 1 00010000:00000001:1.0:1773941498.735784:0:14663:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941498.735784:0:14663:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941498.735785:0:14663:0:(ldlm_lock.c:302:ldlm_lock_add_to_lru()) Process entered 00010000:00000001:1.0:1773941498.735786:0:14663:0:(ldlm_lock.c:306:ldlm_lock_add_to_lru()) Process leaving 00010000:00010000:1.0:1773941498.735788:0:14663:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-MDT0000-mdc-ffff988d08f0a000 lock: ffff988d00030000/0x8e86767622e28c6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x0 nid: local remote: 0xd03f0241a6d949fd expref: -99 pid: 14647 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00010000:00000001:1.0:1773941498.735792:0:14663:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:1.0:1773941498.735793:0:14663:0:(ldlm_request.c:2131:ldlm_cancel_lru()) Process entered 00010000:00000001:1.0:1773941498.735794:0:14663:0:(ldlm_request.c:1927:ldlm_prepare_lru_list()) Process entered 00010000:00000001:1.0:1773941498.735796:0:14663:0:(ldlm_request.c:2099:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941498.735797:0:14663:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:1.0:1773941498.735798:0:14663:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941498.735799:0:14663:0:(ldlm_request.c:2141:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941498.735801:0:14663:0:(ldlm_pool.c:497:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941498.735802:0:14663:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941498.735803:0:14663:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941498.735804:0:14663:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:1.0:1773941498.735805:0:14663:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1773941498.735806:0:14663:0:(client.c:2844:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000080:00000001:1.0:1773941498.735808:0:14663:0:(file.c:6244:ll_merge_md_attr()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941498.735810:0:14663:0:(obd_sysfs.c:873:encode_fn_len()) Process filename at ffffb6ec81cafd08 00000080:00000008:1.0:1773941498.735811:0:14663:0:(file.c:6469:ll_getattr_dentry()) COMPLETED file /:[0x200000007:0x1:0x0](ffff988d20529988), request_mask 2047, flags 0, foreign 0 00000080:00000001:1.0:1773941498.736232:0:14663:0:(llite_lib.c:4092:ll_show_options()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:3.0:1773941498.742171:0:14663:0:(llite_lib.c:1002:ll_kill_super()) Process entered 00000080:00000001:3.0:1773941498.742175:0:14663:0:(llite_lib.c:1023:ll_kill_super()) Process leaving 00000080:00000001:3.0:1773941498.742183:0:14663:0:(dcache.c:39:ll_release()) Process entered 00000080:00000001:3.0:1773941498.742188:0:14663:0:(dcache.c:48:ll_release()) Process leaving 00000080:00000001:3.0:1773941498.742200:0:14663:0:(llite_lib.c:3261:ll_delete_inode()) Process entered 00000080:00000001:3.0:1773941498.742203:0:14663:0:(llite_lib.c:1948:ll_clear_inode()) Process entered 00000080:00200000:3.0:1773941498.742205:0:14663:0:(llite_lib.c:1951:ll_clear_inode()) VFS Op:inode=[0x200000007:0x1:0x0](ffff988d20529988) 00800000:00000001:3.0:1773941498.742230:0:14663:0:(lmv_obd.c:1610:lmv_null_inode()) Process entered 00800000:00000002:3.0:1773941498.742232:0:14663:0:(lmv_obd.c:1612:lmv_null_inode()) CBDATA for [0x200000007:0x1:0x0] 00000002:00000001:3.0:1773941498.742236:0:14663:0:(mdc_locks.c:166:mdc_null_inode()) Process entered 00010000:00000040:3.0:1773941498.742241:0:14663:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff988d21ec1800 count: 2 00010000:00000001:3.0:1773941498.742244:0:14663:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00000002:00000001:3.0:1773941498.742246:0:14663:0:(mdc_locks.c:180:mdc_null_inode()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:3.0:1773941498.742248:0:14663:0:(mdc_locks.c:166:mdc_null_inode()) Process entered 00000002:00000001:3.0:1773941498.742250:0:14663:0:(mdc_locks.c:173:mdc_null_inode()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:3.0:1773941498.742251:0:14663:0:(lmv_obd.c:1622:lmv_null_inode()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:3.0:1773941498.742253:0:14663:0:(xattr_cache.c:319:ll_xattr_cache_destroy()) Process entered 00000080:00000001:3.0:1773941498.742255:0:14663:0:(xattr_cache.c:300:ll_xattr_cache_destroy_locked()) Process entered 00000080:00000001:3.0:1773941498.742257:0:14663:0:(xattr_cache.c:303:ll_xattr_cache_destroy_locked()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:3.0:1773941498.742259:0:14663:0:(xattr_cache.c:325:ll_xattr_cache_destroy()) Process leaving (rc=0 : 0 : 0) 00800000:00000002:3.0:1773941498.742261:0:14663:0:(lmv_obd.c:3892:lmv_stripe_object_put()) put ffff988d10872600 1 00800000:00000010:3.0:1773941498.742264:0:14663:0:(lmv_obd.c:3877:lmv_stripe_object_free()) kfreed 'lsm_obj': 56 at ffff988d10872600. 00000080:00000001:3.0:1773941498.742268:0:14663:0:(llite_lib.c:1999:ll_clear_inode()) Process leaving 00000080:00000001:3.0:1773941498.742270:0:14663:0:(llite_lib.c:3280:ll_delete_inode()) Process leaving 00000080:00000001:3.0:1773941498.742276:0:14663:0:(llite_lib.c:1565:ll_put_super()) Process entered 00000080:00200000:3.0:1773941498.742278:0:14663:0:(llite_lib.c:1572:ll_put_super()) VFS Op: cfg_instance lustre-client-ffff988d08f0a000 (sb ffff988d08f0a000) 00000020:00000001:3.0:1773941498.742282:0:14663:0:(obd_mount.c:116:lustre_end_log()) Process entered 00000020:00000010:3.0:1773941498.742286:0:14663:0:(obd_mount.c:126:lustre_end_log()) kmalloced '(lcfg)': 176 at ffff988d1837de40. 00000020:00000001:3.0:1773941498.742288:0:14663:0:(obd_class.h:681:obd_process_config()) Process entered 00000020:00000010:3.0:1773941498.742292:0:14663:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d02cbc200. 00000020:00000010:3.0:1773941498.742297:0:14663:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d02cbda00. 00000040:00000010:3.0:1773941498.742301:0:14663:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d1817c600. 00000020:00000010:3.0:1773941498.742305:0:14663:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d07233000. 00000020:00000010:3.0:1773941498.742310:0:14663:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d1817c200. 00040000:00000010:3.0:1773941498.742314:0:14663:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d1a2ec800. 00040000:00000010:3.0:1773941498.742318:0:14663:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d1817cc00. 10000000:00000001:3.0:1773941498.742322:0:14663:0:(mgc_request.c:1932:mgc_process_config()) Process entered 10000000:00000001:3.0:1773941498.742324:0:14663:0:(mgc_request_server.c:668:mgc_process_config_server()) Process entered 10000000:00000001:3.0:1773941498.742327:0:14663:0:(mgc_request.c:493:config_log_end()) Process entered 10000000:00000001:3.0:1773941498.742328:0:14663:0:(mgc_request.c:159:config_log_find()) Process entered 10000000:00000001:3.0:1773941498.742330:0:14663:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941498.742332:0:14663:0:(mgc_request.c:108:config_log_get()) log lustre-client (ffff988d1817c000) refs 3 10000000:00000001:3.0:1773941498.742334:0:14663:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941498.742341:0:14663:0:(mgc_request.c:177:config_log_find()) Process leaving (rc=18446630330494861312 : -113743214690304 : ffff988d1817c000) 10000000:00000001:3.0:1773941498.742343:0:14663:0:(mgc_request.c:461:config_mark_cld_stop_nolock()) Process entered 10000000:00000040:3.0:1773941498.742345:0:14663:0:(mgc_request.c:467:config_mark_cld_stop_nolock()) lockh 0x8e86767622e286b 00010000:00000001:3.0:1773941498.742347:0:14663:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1773941498.742348:0:14663:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941498.742350:0:14663:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d0168a600 refcount=2 00000020:00000001:3.0:1773941498.742352:0:14663:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330114287104 : -113743595264512 : ffff988d0168a600) 00010000:00000001:3.0:1773941498.742354:0:14663:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330114287104 : -113743595264512 : ffff988d0168a600) 00010000:00000001:3.0:1773941498.742357:0:14663:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941498.742360:0:14663:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941498.742364:0:14663:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d0168a600/0x8e86767622e286b lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000000000000 nid: local remote: 0xd03f0241a6d949cc expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941498.742370:0:14663:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941498.742371:0:14663:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941498.742372:0:14663:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1773941498.742374:0:14663:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941498.742375:0:14663:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d0168a600 refcount=3 00000020:00000001:3.0:1773941498.742377:0:14663:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330114287104 : -113743595264512 : ffff988d0168a600) 00010000:00000001:3.0:1773941498.742379:0:14663:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330114287104 : -113743595264512 : ffff988d0168a600) 00010000:00000001:3.0:1773941498.742381:0:14663:0:(ldlm_lock.c:950:ldlm_lock_decref_and_cancel()) Process entered 00010000:00010000:3.0:1773941498.742383:0:14663:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d0168a600/0x8e86767622e286b lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0xd03f0241a6d949cc expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941498.742387:0:14663:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:3.0:1773941498.742389:0:14663:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d0168a600/0x8e86767622e286b lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000400000000 nid: local remote: 0xd03f0241a6d949cc expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941498.742393:0:14663:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941498.742393:0:14663:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:3.0:1773941498.742395:0:14663:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d0168a600/0x8e86767622e286b lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11000400000000 nid: local remote: 0xd03f0241a6d949cc expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941498.742407:0:14663:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941498.742408:0:14663:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941498.742410:0:14663:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000010:3.0:1773941498.742413:0:14663:0:(ldlm_lockd.c:2249:ldlm_bl_to_thread()) kmalloced '(blwi)': 192 at ffff988d1837d3c0. 00010000:00000001:3.0:1773941498.742415:0:14663:0:(ldlm_lockd.c:2159:__ldlm_bl_to_thread()) Process entered 00010000:00010000:3.0:1773941498.742422:0:14663:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/1 locks to regular blp list, 1 blwis in pool 00010000:00000001:3.0:1773941498.742423:0:14663:0:(ldlm_lockd.c:2198:__ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941498.742425:0:14663:0:(ldlm_lockd.c:2254:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941498.742426:0:14663:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:3.0:1773941498.742427:0:14663:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941498.742428:0:14663:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 10000000:00000001:3.0:1773941498.742429:0:14663:0:(mgc_request.c:461:config_mark_cld_stop_nolock()) Process entered 10000000:00000040:3.0:1773941498.742430:0:14663:0:(mgc_request.c:467:config_mark_cld_stop_nolock()) lockh 0x8e86767622e2895 00010000:00000001:3.0:1773941498.742431:0:14663:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1773941498.742432:0:14663:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941498.742434:0:14663:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089dfc00 refcount=2 00000020:00000001:3.0:1773941498.742435:0:14663:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235223040 : -113743474328576 : ffff988d089dfc00) 00010000:00000001:3.0:1773941498.742437:0:14663:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235223040 : -113743474328576 : ffff988d089dfc00) 00010000:00000001:3.0:1773941498.742438:0:14663:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941498.742439:0:14663:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941498.742441:0:14663:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089dfc00/0x8e86767622e2895 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xd03f0241a6d949d3 expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941498.742445:0:14663:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941498.742446:0:14663:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941498.742447:0:14663:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1773941498.742448:0:14663:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941498.742449:0:14663:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089dfc00 refcount=3 00000020:00000001:3.0:1773941498.742450:0:14663:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235223040 : -113743474328576 : ffff988d089dfc00) 00010000:00000001:3.0:1773941498.742452:0:14663:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235223040 : -113743474328576 : ffff988d089dfc00) 00010000:00000001:3.0:1773941498.742453:0:14663:0:(ldlm_lock.c:950:ldlm_lock_decref_and_cancel()) Process entered 00010000:00010000:3.0:1773941498.742455:0:14663:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089dfc00/0x8e86767622e2895 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0xd03f0241a6d949d3 expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941498.742464:0:14663:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:3.0:1773941498.742465:0:14663:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089dfc00/0x8e86767622e2895 lrc: 3/1,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000400000000 nid: local remote: 0xd03f0241a6d949d3 expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941498.742469:0:14663:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941498.742470:0:14663:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:3.0:1773941498.742471:0:14663:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d089dfc00/0x8e86767622e2895 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x11000400000000 nid: local remote: 0xd03f0241a6d949d3 expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941498.742475:0:14663:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941498.742476:0:14663:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941498.742477:0:14663:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000010:3.0:1773941498.742479:0:14663:0:(ldlm_lockd.c:2249:ldlm_bl_to_thread()) kmalloced '(blwi)': 192 at ffff988d1837d9c0. 00010000:00000001:3.0:1773941498.742481:0:14663:0:(ldlm_lockd.c:2159:__ldlm_bl_to_thread()) Process entered 00010000:00010000:3.0:1773941498.742490:0:14663:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 00010000:00000001:3.0:1773941498.742492:0:14663:0:(ldlm_lockd.c:2198:__ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941498.742493:0:14663:0:(ldlm_lockd.c:2254:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941498.742494:0:14663:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:3.0:1773941498.742496:0:14663:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941498.742497:0:14663:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 10000000:00000001:3.0:1773941498.742499:0:14663:0:(mgc_request.c:117:config_log_put()) Process entered 00000020:00000001:1.0:1773941498.742499:0:14589:0:(genops.c:1914:obd_stale_export_get()) Process entered 10000000:00000040:3.0:1773941498.742502:0:14663:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff988d1817c800) refs 2 00000020:00000001:1.0:1773941498.742504:0:14589:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941498.742505:0:14663:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:3.0:1773941498.742509:0:14663:0:(mgc_request.c:461:config_mark_cld_stop_nolock()) Process entered 00010000:00010000:1.0:1773941498.742510:0:14589:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 10000000:00000040:3.0:1773941498.742511:0:14663:0:(mgc_request.c:467:config_mark_cld_stop_nolock()) lockh 0x8e86767622e289c 00010000:00000001:3.0:1773941498.742515:0:14663:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:1.0:1773941498.742515:0:14589:0:(ldlm_lockd.c:2874:ldlm_bl_thread_blwi()) Process entered 00000020:00000001:3.0:1773941498.742517:0:14663:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941498.742518:0:14663:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d0168b400 refcount=2 00000020:00000001:3.0:1773941498.742520:0:14663:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330114290688 : -113743595260928 : ffff988d0168b400) 00010000:00000001:3.0:1773941498.742526:0:14663:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330114290688 : -113743595260928 : ffff988d0168b400) 00010000:00000001:1.0:1773941498.742526:0:14589:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00000001:3.0:1773941498.742530:0:14663:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941498.742532:0:14663:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941498.742532:0:14589:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d0168a600/0x8e86767622e286b lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000400000000 nid: local remote: 0xd03f0241a6d949cc expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941498.742543:0:14589:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d0168a600 already unused, calling callback (ffffffffc12b4c20) 00010000:00010000:3.0:1773941498.742544:0:14663:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d0168b400/0x8e86767622e289c lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000000000000 nid: local remote: 0xd03f0241a6d949da expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:1.0:1773941498.742548:0:14589:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00010000:00000001:3.0:1773941498.742553:0:14663:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 10000000:00010000:1.0:1773941498.742553:0:14589:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d0168a600/0x8e86767622e286b lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000400000000 nid: local remote: 0xd03f0241a6d949cc expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941498.742556:0:14663:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941498.742559:0:14663:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1773941498.742562:0:14663:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941498.742565:0:14663:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d0168b400 refcount=3 00010000:00000001:1.0:1773941498.742569:0:14589:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000020:00000001:3.0:1773941498.742570:0:14663:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330114290688 : -113743595260928 : ffff988d0168b400) 00010000:00000001:1.0:1773941498.742573:0:14589:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:3.0:1773941498.742575:0:14663:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330114290688 : -113743595260928 : ffff988d0168b400) 00000020:00000001:1.0:1773941498.742577:0:14589:0:(lustre_handles.c:113:class_handle2object()) Process entered 00010000:00000001:3.0:1773941498.742580:0:14663:0:(ldlm_lock.c:950:ldlm_lock_decref_and_cancel()) Process entered 00000020:00000040:1.0:1773941498.742582:0:14589:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d0168a600 refcount=3 00010000:00010000:3.0:1773941498.742585:0:14663:0:(ldlm_lock.c:954:ldlm_lock_decref_and_cancel()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d0168b400/0x8e86767622e289c lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0xd03f0241a6d949da expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000020:00000001:1.0:1773941498.742587:0:14589:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330114287104 : -113743595264512 : ffff988d0168a600) 00010000:00000001:1.0:1773941498.742594:0:14589:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330114287104 : -113743595264512 : ffff988d0168a600) 00010000:00000001:1.0:1773941498.742601:0:14589:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00000001:3.0:1773941498.742602:0:14663:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941498.742606:0:14589:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d0168a600/0x8e86767622e286b lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1008400000000 nid: local remote: 0xd03f0241a6d949cc expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941498.742607:0:14663:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d0168b400/0x8e86767622e289c lrc: 3/1,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000400000000 nid: local remote: 0xd03f0241a6d949da expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941498.742625:0:14663:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941498.742628:0:14663:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 10000000:00000001:1.0:1773941498.742629:0:14589:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00010000:00010000:3.0:1773941498.742630:0:14663:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d0168b400/0x8e86767622e289c lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x11000400000000 nid: local remote: 0xd03f0241a6d949da expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00010000:1.0:1773941498.742634:0:14589:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d0168a600/0x8e86767622e286b lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1009400000000 nid: local remote: 0xd03f0241a6d949cc expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941498.742638:0:14663:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941498.742642:0:14663:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 10000000:01000000:1.0:1773941498.742644:0:14589:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00000001:3.0:1773941498.742645:0:14663:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 10000000:01000000:1.0:1773941498.742649:0:14589:0:(mgc_request.c:944:mgc_blocking_ast()) log lustre-client: stopping, won't requeue 00010000:00000010:3.0:1773941498.742650:0:14663:0:(ldlm_lockd.c:2249:ldlm_bl_to_thread()) kmalloced '(blwi)': 192 at ffff988d1837db40. 10000000:00000001:1.0:1773941498.742651:0:14589:0:(mgc_request.c:117:config_log_put()) Process entered 00010000:00000001:3.0:1773941498.742655:0:14663:0:(ldlm_lockd.c:2159:__ldlm_bl_to_thread()) Process entered 10000000:00000040:1.0:1773941498.742655:0:14589:0:(mgc_request.c:123:config_log_put()) log lustre-client(ffff988d1817c000) refs 3 10000000:00000001:1.0:1773941498.742659:0:14589:0:(mgc_request.c:147:config_log_put()) Process leaving 00010000:00010000:3.0:1773941498.742660:0:14663:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 1/2 locks to regular blp list, 2 blwis in pool 10000000:00000001:1.0:1773941498.742661:0:14589:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941498.742662:0:14588:0:(genops.c:1914:obd_stale_export_get()) Process entered 00010000:00000001:3.0:1773941498.742665:0:14663:0:(ldlm_lockd.c:2198:__ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941498.742668:0:14588:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941498.742669:0:14663:0:(ldlm_lockd.c:2254:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941498.742669:0:14589:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941498.742674:0:14589:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:3.0:1773941498.742675:0:14663:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00010000:2.0:1773941498.742676:0:14588:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 1 total in blp. (1 blwis in pool) 00010000:00000001:1.0:1773941498.742677:0:14589:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:3.0:1773941498.742678:0:14663:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941498.742681:0:14589:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941498.742682:0:14663:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941498.742685:0:14589:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941498.742691:0:14663:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:3.0:1773941498.742694:0:14663:0:(mgc_request.c:123:config_log_put()) log params(ffff988d1817d200) refs 2 00000020:00000040:1.0:1773941498.742696:0:14589:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d0168a600 with handle 0x8e86767622e286b from hash 10000000:00000001:3.0:1773941498.742697:0:14663:0:(mgc_request.c:147:config_log_put()) Process leaving 00010000:00000001:1.0:1773941498.742700:0:14589:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 10000000:00000001:3.0:1773941498.742701:0:14663:0:(mgc_request.c:117:config_log_put()) Process entered 00010000:00000001:1.0:1773941498.742704:0:14589:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 10000000:00000001:3.0:1773941498.742705:0:14663:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941498.742709:0:14663:0:(mgc_request.c:117:config_log_put()) Process entered 00010000:00000001:1.0:1773941498.742709:0:14589:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941498.742713:0:14589:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 10000000:00000040:3.0:1773941498.742714:0:14663:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4200) refs 2 00010000:00000001:1.0:1773941498.742717:0:14589:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 10000000:00000001:3.0:1773941498.742718:0:14663:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:3.0:1773941498.742721:0:14663:0:(mgc_request.c:117:config_log_put()) Process entered 00010000:00000001:1.0:1773941498.742721:0:14589:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 10000000:00000001:3.0:1773941498.742722:0:14663:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941498.742726:0:14663:0:(mgc_request.c:117:config_log_put()) Process entered 00010000:00000001:1.0:1773941498.742727:0:14589:0:(ldlm_request.c:2242:ldlm_cli_cancel_list()) Process entered 10000000:00000040:3.0:1773941498.742728:0:14663:0:(mgc_request.c:123:config_log_put()) log lustre-client(ffff988d1817c000) refs 2 00010000:00000001:1.0:1773941498.742731:0:14589:0:(ldlm_request.c:1396:ldlm_cli_cancel_req()) Process entered 10000000:00000001:3.0:1773941498.742732:0:14663:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:3.0:1773941498.742735:0:14663:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:3.0:1773941498.742735:0:14663:0:(mgc_request.c:123:config_log_put()) log lustre-client(ffff988d1817c000) refs 1 10000000:01000000:3.0:1773941498.742737:0:14663:0:(mgc_request.c:130:config_log_put()) dropping config log lustre-client 10000000:00000001:3.0:1773941498.742738:0:14663:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941498.742739:0:14663:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941498.742740:0:14663:0:(mgc_request.c:117:config_log_put()) Process entered 00000100:00000010:1.0:1773941498.742740:0:14589:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c2a00. 10000000:00000001:3.0:1773941498.742743:0:14663:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941498.742745:0:14663:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941498.742746:0:14663:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941498.742748:0:14663:0:(mgc_request.c:117:config_log_put()) Process entered 00000020:00000040:1.0:1773941498.742748:0:14589:0:(genops.c:1229:class_import_get()) import ffff988d035c4800 refcount=4 obd=MGC192.168.204.114@tcp 10000000:00000001:3.0:1773941498.742750:0:14663:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941498.742752:0:14663:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941498.742754:0:14663:0:(mgc_request.c:120:config_log_put()) Process leaving 00000020:00000040:3.0:1773941498.742757:0:14663:0:(genops.c:898:class_export_put()) PUTting export ffff988d08f0d800 : new refcount 5 00000100:00000001:1.0:1773941498.742757:0:14589:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 10000000:00000010:3.0:1773941498.742759:0:14663:0:(mgc_request.c:144:config_log_put()) kfreed 'cld': 286 at ffff988d1817c000. 02000000:00000001:1.0:1773941498.742761:0:14589:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 10000000:00000001:3.0:1773941498.742763:0:14663:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:01000000:3.0:1773941498.742765:0:14663:0:(mgc_request.c:547:config_log_end()) end config log lustre-client (0) 10000000:00000001:3.0:1773941498.742767:0:14663:0:(mgc_request.c:548:config_log_end()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941498.742767:0:14589:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941498.742771:0:14663:0:(mgc_request.c:2026:mgc_process_config()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941498.742776:0:14663:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d02cbda00. 02000000:00000010:1.0:1773941498.742779:0:14589:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d08a4e200. 00000040:00000010:3.0:1773941498.742780:0:14663:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d1817c600. 00000020:00000010:3.0:1773941498.742786:0:14663:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d07233000. 00000020:00000010:3.0:1773941498.742790:0:14663:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d1817c200. 00000100:00000001:1.0:1773941498.742790:0:14589:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00040000:00000010:3.0:1773941498.742796:0:14663:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d1a2ec800. 00010000:00000001:1.0:1773941498.742797:0:14589:0:(ldlm_request.c:1348:ldlm_cancel_pack()) Process entered 00040000:00000010:3.0:1773941498.742800:0:14663:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d1817cc00. 00000020:00000010:3.0:1773941498.742805:0:14663:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d02cbc200. 00010000:00010000:1.0:1773941498.742806:0:14589:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: MGC192.168.204.114@tcp lock: ffff988d0168a600/0x8e86767622e286b lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5809400000020 nid: local remote: 0xd03f0241a6d949cc expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000020:00000001:3.0:1773941498.742815:0:14663:0:(obd_class.h:696:obd_process_config()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941498.742821:0:14663:0:(obd_mount.c:131:lustre_end_log()) kfreed 'lcfg': 176 at ffff988d1837de40. 00010000:00010000:1.0:1773941498.742821:0:14589:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00000020:00000001:3.0:1773941498.742827:0:14663:0:(obd_mount.c:132:lustre_end_log()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941498.742827:0:14589:0:(ldlm_request.c:1380:ldlm_cancel_pack()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1773941498.742833:0:14663:0:(obd_mount.c:116:lustre_end_log()) Process entered 00000020:00000010:3.0:1773941498.742836:0:14663:0:(obd_mount.c:126:lustre_end_log()) kmalloced '(lcfg)': 168 at ffff988d1837de40. 00000100:00000001:1.0:1773941498.742837:0:14589:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000020:00000001:3.0:1773941498.742838:0:14663:0:(obd_class.h:681:obd_process_config()) Process entered 00000020:00000010:3.0:1773941498.742841:0:14663:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d1817cc00. 00000100:00000001:1.0:1773941498.742844:0:14589:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941498.742845:0:14663:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d1817c200. 00000040:00000010:3.0:1773941498.742848:0:14663:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d1817c600. 00000020:00000010:3.0:1773941498.742853:0:14663:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d07233000. 00000020:00000010:3.0:1773941498.742855:0:14663:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d1817c000. 00040000:00000010:3.0:1773941498.742858:0:14663:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d1a2ec800. 00000100:00000040:1.0:1773941498.742862:0:14589:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fb7523b4] to pc [ptlrpcd_00_01+1] req@ffff988d203c2a00 x1860112291475584/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl New:QU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00040000:00000010:3.0:1773941498.742863:0:14663:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d1817ce00. 10000000:00000001:3.0:1773941498.742868:0:14663:0:(mgc_request.c:1932:mgc_process_config()) Process entered 10000000:00000001:3.0:1773941498.742871:0:14663:0:(mgc_request_server.c:668:mgc_process_config_server()) Process entered 10000000:00000001:3.0:1773941498.742874:0:14663:0:(mgc_request.c:493:config_log_end()) Process entered 10000000:00000001:3.0:1773941498.742877:0:14663:0:(mgc_request.c:159:config_log_find()) Process entered 10000000:00000001:3.0:1773941498.742881:0:14663:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941498.742884:0:14663:0:(mgc_request.c:108:config_log_get()) log params (ffff988d1817d200) refs 2 10000000:00000001:3.0:1773941498.742887:0:14663:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941498.742889:0:14663:0:(mgc_request.c:177:config_log_find()) Process leaving (rc=18446630330494865920 : -113743214685696 : ffff988d1817d200) 10000000:00000001:3.0:1773941498.742893:0:14663:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:3.0:1773941498.742895:0:14663:0:(mgc_request.c:123:config_log_put()) log params(ffff988d1817d200) refs 2 10000000:00000001:3.0:1773941498.742897:0:14663:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:3.0:1773941498.742900:0:14663:0:(mgc_request.c:511:config_log_end()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941498.742900:0:14589:0:(ldlm_request.c:1464:ldlm_cli_cancel_req()) Process leaving via out (rc=0 : 0 : 0x0) 10000000:00000001:3.0:1773941498.742905:0:14663:0:(mgc_request.c:2026:mgc_process_config()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941498.742907:0:14589:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000020:00000010:3.0:1773941498.742910:0:14663:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d1817c200. 00010000:00000001:1.0:1773941498.742911:0:14589:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941498.742915:0:14589:0:(ldlm_request.c:2290:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00000040:00000010:3.0:1773941498.742917:0:14663:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d1817c600. 00010000:00000001:1.0:1773941498.742919:0:14589:0:(ldlm_request.c:1689:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941498.742922:0:14663:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d07233000. 10000000:00000001:1.0:1773941498.742924:0:14589:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941498.742929:0:14663:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d1817c000. 00010000:00010000:1.0:1773941498.742931:0:14589:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d0168a600/0x8e86767622e286b lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5809400000020 nid: local remote: 0xd03f0241a6d949cc expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00040000:00000010:3.0:1773941498.742934:0:14663:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d1a2ec800. 00040000:00000010:3.0:1773941498.742938:0:14663:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d1817ce00. 00000020:00000010:3.0:1773941498.742942:0:14663:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d1817cc00. 00000020:00000001:3.0:1773941498.742945:0:14663:0:(obd_class.h:696:obd_process_config()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941498.742947:0:14663:0:(obd_mount.c:131:lustre_end_log()) kfreed 'lcfg': 168 at ffff988d1837de40. 00000020:00000001:3.0:1773941498.742949:0:14663:0:(obd_mount.c:132:lustre_end_log()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941498.742958:0:14589:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000080:00000001:3.0:1773941498.742980:0:14663:0:(llite_lib.c:983:client_common_put_super()) Process entered 00000080:00000001:3.0:1773941498.742983:0:14663:0:(vvp_dev.c:313:cl_sb_fini()) Process entered 00000020:00000001:3.0:1773941498.742985:0:14663:0:(cl_object.c:762:cl_env_obtain()) Process entered 00010000:00010000:1.0:1773941498.742987:0:14589:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d0168a600/0x8e86767622e286b lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5809400000020 nid: local remote: 0xd03f0241a6d949cc expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000020:00000010:3.0:1773941498.742990:0:14663:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff988d08a3f3f0. 00000020:00000010:3.0:1773941498.742995:0:14663:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d1817cc00. 00000020:00000010:3.0:1773941498.743000:0:14663:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d1817ce00. 00000020:00000010:1.0:1773941498.743000:0:14589:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 128 at ffff988d0716be80. 00000020:00000010:3.0:1773941498.743008:0:14663:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d111f7540. 00010000:00000001:1.0:1773941498.743010:0:14589:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941498.743013:0:14589:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000008:00000010:3.0:1773941498.743014:0:14663:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff988d113f9650. 00010000:00000010:1.0:1773941498.743015:0:14589:0:(ldlm_lockd.c:2913:ldlm_bl_thread_blwi()) kfreed 'blwi': 192 at ffff988d1837d3c0. 00020000:00000010:3.0:1773941498.743020:0:14663:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff988d20218d80. 00010000:00000001:2.0:1773941498.743020:0:14667:0:(ldlm_lockd.c:2959:ldlm_bl_thread_main()) Process entered 00010000:00000001:1.0:1773941498.743021:0:14589:0:(ldlm_lockd.c:2917:ldlm_bl_thread_blwi()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941498.743027:0:14589:0:(genops.c:1914:obd_stale_export_get()) Process entered 00010000:00000010:2.0:1773941498.743028:0:14667:0:(ldlm_lockd.c:2961:ldlm_bl_thread_main()) kmalloced '(env)': 56 at ffff988d05437640. 00008000:00000010:3.0:1773941498.743029:0:14663:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d107f2de8. 00000020:00000001:1.0:1773941498.743029:0:14589:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941498.743034:0:14589:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 1 locks of 0 total in blp. (0 blwis in pool) 00010000:00000001:1.0:1773941498.743037:0:14589:0:(ldlm_lockd.c:2874:ldlm_bl_thread_blwi()) Process entered 00000080:00000010:3.0:1773941498.743038:0:14663:0:(vvp_dev.c:68:ll_thread_key_init()) slab-alloced 'lti': 168 at ffff988d181e7930. 00000020:00000010:2.0:1773941498.743039:0:14667:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d0350ea00. 00010000:00000001:1.0:1773941498.743039:0:14589:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941498.743044:0:14589:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d0168b400/0x8e86767622e289c lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000400000000 nid: local remote: 0xd03f0241a6d949da expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00000010:3.0:1773941498.743046:0:14663:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff988d054abb50. 00000020:00000010:2.0:1773941498.743049:0:14667:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d0350e800. 00010000:00010000:1.0:1773941498.743053:0:14589:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d0168b400 already unused, calling callback (ffffffffc12b4c20) 00000020:00000010:3.0:1773941498.743055:0:14663:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d1817c000. 10000000:00000001:1.0:1773941498.743056:0:14589:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000020:00000010:2.0:1773941498.743057:0:14667:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d05449000. 10000000:00010000:1.0:1773941498.743060:0:14589:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d0168b400/0x8e86767622e289c lrc: 2/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1000400000000 nid: local remote: 0xd03f0241a6d949da expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000008:00000010:3.0:1773941498.743061:0:14663:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff988d035ca9a0. 00000020:00000010:2.0:1773941498.743066:0:14667:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff988d05400000. 00020000:00000010:3.0:1773941498.743068:0:14663:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff988d113f6880. 00010000:00000001:1.0:1773941498.743068:0:14589:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941498.743072:0:14589:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941498.743074:0:14589:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000010:2.0:1773941498.743075:0:14667:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d0350fc00. 00008000:00000010:3.0:1773941498.743076:0:14663:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff988d209199e8. 00000020:00000040:1.0:1773941498.743076:0:14589:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d0168b400 refcount=3 00000020:00000001:1.0:1773941498.743081:0:14589:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330114290688 : -113743595260928 : ffff988d0168b400) 00000080:00000010:3.0:1773941498.743082:0:14663:0:(vvp_dev.c:94:vvp_session_key_init()) slab-alloced 'session': 240 at ffff988d071534b0. 00010000:00000001:1.0:1773941498.743084:0:14589:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330114290688 : -113743595260928 : ffff988d0168b400) 00000020:00000001:3.0:1773941498.743089:0:14663:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446630330235614200 : -113743473937416 : ffff988d08a3f3f8) 00010000:00000001:1.0:1773941498.743089:0:14589:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 80000000:00000010:2.0:1773941498.743090:0:14667:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d08bbbd20. 00010000:00010000:1.0:1773941498.743094:0:14589:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d0168b400/0x8e86767622e289c lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1008400000000 nid: local remote: 0xd03f0241a6d949da expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000020:00001000:3.0:1773941498.743095:0:14663:0:(cl_object.c:831:cl_env_get()) 1@ffff988d08a3f3f0 40000000:00000010:2.0:1773941498.743098:0:14667:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff988d054375c0. 00100000:00000010:2.0:1773941498.743107:0:14667:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff988d05449800. 00040000:00000010:2.0:1773941498.743116:0:14667:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d217a8d00. 10000000:00000001:1.0:1773941498.743117:0:14589:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941498.743121:0:14589:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d0168b400/0x8e86767622e289c lrc: 3/0,0 mode: CR/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x1009400000000 nid: local remote: 0xd03f0241a6d949da expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00040000:00000010:2.0:1773941498.743122:0:14667:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d0350f200. 00000020:00000040:3.0:1773941498.743128:0:14663:0:(obd_config.c:1058:class_decref()) Decref lustre-clilmv-ffff988d08f0a000 (ffff988d089a8000) now 5 - lov 10000000:01000000:1.0:1773941498.743130:0:14589:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x736d61726170:0x3:0x0].0x0 (params) 00002000:00000010:2.0:1773941498.743131:0:14667:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff988d035a4400. 10000000:01000000:1.0:1773941498.743134:0:14589:0:(mgc_request.c:944:mgc_blocking_ast()) log params: stopping, won't requeue 10000000:00000001:1.0:1773941498.743137:0:14589:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941498.743138:0:14589:0:(mgc_request.c:123:config_log_put()) log params(ffff988d1817d200) refs 1 00000020:00000010:2.0:1773941498.743139:0:14667:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff988d05437340. 10000000:01000000:1.0:1773941498.743141:0:14589:0:(mgc_request.c:130:config_log_put()) dropping config log params 10000000:00000001:1.0:1773941498.743144:0:14589:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941498.743145:0:14589:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941498.743147:0:14589:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941498.743148:0:14589:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941498.743149:0:14589:0:(mgc_request.c:117:config_log_put()) Process entered 00000020:00000001:2.0:1773941498.743151:0:14667:0:(genops.c:1914:obd_stale_export_get()) Process entered 10000000:00000001:1.0:1773941498.743151:0:14589:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941498.743153:0:14589:0:(mgc_request.c:117:config_log_put()) Process entered 00000020:00000001:2.0:1773941498.743155:0:14667:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941498.743156:0:14589:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941498.743158:0:14589:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941498.743161:0:14589:0:(mgc_request.c:120:config_log_put()) Process leaving 00010000:00010000:2.0:1773941498.743162:0:14667:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000040:1.0:1773941498.743163:0:14589:0:(genops.c:898:class_export_put()) PUTting export ffff988d08f0d800 : new refcount 4 10000000:00000010:1.0:1773941498.743167:0:14589:0:(mgc_request.c:144:config_log_put()) kfreed 'cld': 279 at ffff988d1817d200. 00010000:00000001:2.0:1773941498.743169:0:14588:0:(ldlm_lockd.c:2874:ldlm_bl_thread_blwi()) Process entered 10000000:00000001:1.0:1773941498.743170:0:14589:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:1.0:1773941498.743174:0:14589:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941498.743175:0:14588:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00000001:1.0:1773941498.743178:0:14589:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00020000:00000001:3.0:1773941498.743179:0:14663:0:(lovsub_dev.c:57:lovsub_device_fini()) Process entered 00010000:00010000:2.0:1773941498.743181:0:14588:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d089dfc00/0x8e86767622e2895 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000400000000 nid: local remote: 0xd03f0241a6d949d3 expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00020000:00000001:3.0:1773941498.743182:0:14663:0:(lovsub_dev.c:61:lovsub_device_fini()) Process leaving (rc=18446630330146750592 : -113743562801024 : ffff988d03580080) 00010000:00000001:1.0:1773941498.743182:0:14589:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941498.743186:0:14589:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941498.743189:0:14589:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941498.743193:0:14589:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941498.743197:0:14589:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d0168b400 with handle 0x8e86767622e289c from hash 00010000:00010000:2.0:1773941498.743199:0:14588:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d089dfc00 already unused, calling callback (ffffffffc12b4c20) 00010000:00000001:1.0:1773941498.743200:0:14589:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941498.743203:0:14589:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 10000000:00000001:2.0:1773941498.743205:0:14588:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00010000:00000001:1.0:1773941498.743206:0:14589:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941498.743209:0:14589:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 10000000:00010000:2.0:1773941498.743211:0:14588:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d089dfc00/0x8e86767622e2895 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1000400000000 nid: local remote: 0xd03f0241a6d949d3 expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941498.743212:0:14589:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941498.743225:0:14589:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00020000:00000010:3.0:1773941498.743231:0:14663:0:(lovsub_dev.c:72:lovsub_device_free()) kfreed 'lsd': 72 at ffff988d183505a0. 00010000:00000001:1.0:1773941498.743237:0:14589:0:(ldlm_request.c:2242:ldlm_cli_cancel_list()) Process entered 00010000:00000001:1.0:1773941498.743240:0:14589:0:(ldlm_request.c:1396:ldlm_cli_cancel_req()) Process entered 00010000:00000001:2.0:1773941498.743244:0:14588:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000100:00000010:1.0:1773941498.743246:0:14589:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c3480. 00010000:00000001:2.0:1773941498.743248:0:14588:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000040:1.0:1773941498.743250:0:14589:0:(genops.c:1229:class_import_get()) import ffff988d035c4800 refcount=5 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941498.743251:0:14588:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:1.0:1773941498.743255:0:14589:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000020:00000040:2.0:1773941498.743256:0:14588:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089dfc00 refcount=3 02000000:00000001:1.0:1773941498.743259:0:14589:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:2.0:1773941498.743261:0:14588:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235223040 : -113743474328576 : ffff988d089dfc00) 02000000:00000001:1.0:1773941498.743262:0:14589:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1773941498.743266:0:14663:0:(lovsub_dev.c:57:lovsub_device_fini()) Process entered 00010000:00000001:2.0:1773941498.743268:0:14588:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235223040 : -113743474328576 : ffff988d089dfc00) 00020000:00000001:3.0:1773941498.743269:0:14663:0:(lovsub_dev.c:61:lovsub_device_fini()) Process leaving (rc=18446630330146751616 : -113743562800000 : ffff988d03580480) 02000000:00000010:1.0:1773941498.743270:0:14589:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d08a4f800. 00010000:00000001:2.0:1773941498.743274:0:14588:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000100:00000001:1.0:1773941498.743277:0:14589:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941498.743280:0:14588:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d089dfc00/0x8e86767622e2895 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1008400000000 nid: local remote: 0xd03f0241a6d949d3 expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941498.743280:0:14589:0:(ldlm_request.c:1348:ldlm_cancel_pack()) Process entered 00010000:00010000:1.0:1773941498.743286:0:14589:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: MGC192.168.204.114@tcp lock: ffff988d0168b400/0x8e86767622e289c lrc: 2/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5809400000020 nid: local remote: 0xd03f0241a6d949da expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00020000:00000010:3.0:1773941498.743295:0:14663:0:(lovsub_dev.c:72:lovsub_device_free()) kfreed 'lsd': 72 at ffff988d18350720. 10000000:00000001:2.0:1773941498.743297:0:14588:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00010000:00010000:1.0:1773941498.743299:0:14589:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00000001:1.0:1773941498.743302:0:14589:0:(ldlm_request.c:1380:ldlm_cancel_pack()) Process leaving (rc=1 : 1 : 1) 10000000:00010000:2.0:1773941498.743304:0:14588:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d089dfc00/0x8e86767622e2895 lrc: 3/0,0 mode: CR/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x1009400000000 nid: local remote: 0xd03f0241a6d949d3 expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941498.743305:0:14589:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941498.743310:0:14589:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941498.743319:0:14589:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000099ddc7e5] to pc [ptlrpcd_00_00+0] req@ffff988d203c3480 x1860112291475712/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl New:QU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00020000:00000001:3.0:1773941498.743323:0:14663:0:(lovsub_dev.c:57:lovsub_device_fini()) Process entered 00020000:00000001:3.0:1773941498.743326:0:14663:0:(lovsub_dev.c:61:lovsub_device_fini()) Process leaving (rc=18446630330146753408 : -113743562798208 : ffff988d03580b80) 00010000:00000001:1.0:1773941498.743341:0:14589:0:(ldlm_request.c:1464:ldlm_cli_cancel_req()) Process leaving via out (rc=0 : 0 : 0x0) 10000000:01000000:2.0:1773941498.743342:0:14588:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x2:0x0].0x0 (lustre) 00010000:00000001:1.0:1773941498.743346:0:14589:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 10000000:01000000:2.0:1773941498.743349:0:14588:0:(mgc_request.c:944:mgc_blocking_ast()) log lustre-cliir: stopping, won't requeue 00010000:00000001:1.0:1773941498.743349:0:14589:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 10000000:00000001:2.0:1773941498.743354:0:14588:0:(mgc_request.c:117:config_log_put()) Process entered 00010000:00000001:1.0:1773941498.743354:0:14589:0:(ldlm_request.c:2290:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:3.0:1773941498.743356:0:14663:0:(lovsub_dev.c:72:lovsub_device_free()) kfreed 'lsd': 72 at ffff988d183021e0. 10000000:00000040:2.0:1773941498.743359:0:14588:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff988d1817c800) refs 1 00010000:00000001:1.0:1773941498.743359:0:14589:0:(ldlm_request.c:1689:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:3.0:1773941498.743361:0:14663:0:(lov_dev.c:223:lov_device_fini()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941498.743364:0:14589:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 10000000:01000000:2.0:1773941498.743367:0:14588:0:(mgc_request.c:130:config_log_put()) dropping config log lustre-cliir 00010000:00010000:1.0:1773941498.743368:0:14589:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d0168b400/0x8e86767622e289c lrc: 1/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5809400000020 nid: local remote: 0xd03f0241a6d949da expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:2.0:1773941498.743370:0:14588:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941498.743373:0:14588:0:(mgc_request.c:120:config_log_put()) Process leaving 00010000:00000001:1.0:1773941498.743376:0:14589:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 10000000:00000001:2.0:1773941498.743377:0:14588:0:(mgc_request.c:117:config_log_put()) Process entered 00010000:00010000:1.0:1773941498.743379:0:14589:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d0168b400/0x8e86767622e289c lrc: 0/0,0 mode: --/CR res: [0x736d61726170:0x3:0x0].0x0 rrc: 2 type: PLN flags: 0x5809400000020 nid: local remote: 0xd03f0241a6d949da expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:2.0:1773941498.743380:0:14588:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941498.743384:0:14588:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941498.743388:0:14588:0:(mgc_request.c:120:config_log_put()) Process leaving 00010000:00000040:1.0:1773941498.743389:0:14589:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d1a01da00 count: 0 10000000:00000001:2.0:1773941498.743391:0:14588:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941498.743394:0:14588:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941498.743396:0:14588:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941498.743397:0:14588:0:(mgc_request.c:120:config_log_put()) Process leaving 00000020:00000040:2.0:1773941498.743400:0:14588:0:(genops.c:898:class_export_put()) PUTting export ffff988d08f0d800 : new refcount 3 00010000:00000001:1.0:1773941498.743403:0:14589:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 10000000:00000010:2.0:1773941498.743404:0:14588:0:(mgc_request.c:144:config_log_put()) kfreed 'cld': 285 at ffff988d1817c800. 10000000:00000001:2.0:1773941498.743408:0:14588:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:2.0:1773941498.743409:0:14588:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941498.743414:0:14589:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000020:00000010:3.0:1773941498.743415:0:14663:0:(lu_object.c:1212:lu_site_fini()) kvfree 's->ls_bkts': 18432 at ffff988d18b78000. 00010000:00000001:2.0:1773941498.743415:0:14588:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000010:1.0:1773941498.743420:0:14589:0:(ldlm_lockd.c:2913:ldlm_bl_thread_blwi()) kfreed 'blwi': 192 at ffff988d1837db40. 00010000:00000001:2.0:1773941498.743421:0:14588:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1773941498.743424:0:14588:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941498.743427:0:14589:0:(ldlm_lockd.c:2917:ldlm_bl_thread_blwi()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941498.743428:0:14588:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000020:00000010:3.0:1773941498.743429:0:14663:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 192 at ffff988d1837d0c0. 00010000:00000001:2.0:1773941498.743432:0:14588:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941498.743434:0:14589:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000010:3.0:1773941498.743435:0:14663:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff988d054e3800. 00000020:00000040:2.0:1773941498.743437:0:14588:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d089dfc00 with handle 0x8e86767622e2895 from hash 00000020:00000001:1.0:1773941498.743438:0:14589:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941498.743441:0:14588:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1773941498.743446:0:14588:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941498.743446:0:14589:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00000010:3.0:1773941498.743447:0:14663:0:(vvp_dev.c:155:vvp_device_free()) kfreed 'site': 576 at ffff988d111d2000. 00010000:00000001:2.0:1773941498.743448:0:14588:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000020:00000001:1.0:1773941498.743451:0:14589:0:(genops.c:1914:obd_stale_export_get()) Process entered 00010000:00000001:2.0:1773941498.743453:0:14588:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000020:00000001:1.0:1773941498.743454:0:14589:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941498.743456:0:14588:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000080:00000010:3.0:1773941498.743459:0:14663:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff988d181e71f8. 00010000:00000001:2.0:1773941498.743461:0:14588:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00010000:1.0:1773941498.743461:0:14589:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00000010:3.0:1773941498.743463:0:14663:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff988d181e7348. 00000020:00000001:1.0:1773941498.743466:0:14589:0:(genops.c:1914:obd_stale_export_get()) Process entered 00010000:00000001:2.0:1773941498.743467:0:14588:0:(ldlm_request.c:2242:ldlm_cli_cancel_list()) Process entered 00000080:00000010:3.0:1773941498.743468:0:14663:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff988d181e7498. 00000020:00000001:1.0:1773941498.743470:0:14589:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941498.743471:0:14588:0:(ldlm_request.c:1396:ldlm_cli_cancel_req()) Process entered 00000080:00000010:3.0:1773941498.743473:0:14663:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff988d181e7f18. 00010000:00010000:1.0:1773941498.743474:0:14589:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000010:2.0:1773941498.743478:0:14588:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06507100. 00000080:00000010:3.0:1773941498.743479:0:14663:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff988d071530f0. 00000100:00000001:1.0:1773941498.743483:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000080:00000010:3.0:1773941498.743486:0:14663:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff988d071533c0. 00000020:00000040:2.0:1773941498.743486:0:14588:0:(genops.c:1229:class_import_get()) import ffff988d035c4800 refcount=6 obd=MGC192.168.204.114@tcp 00000080:00000010:3.0:1773941498.743489:0:14663:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff988d07153b40. 00000008:00000010:1.0:1773941498.743490:0:2402:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff988d08945940. 00000080:00000010:3.0:1773941498.743491:0:14663:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff988d07153000. 00020000:00000010:1.0:1773941498.743497:0:2402:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff988d180f76c0. 00000100:00000001:2.0:1773941498.743499:0:14588:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941498.743503:0:14588:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 00000080:00000010:1.0:1773941498.743503:0:2402:0:(vvp_dev.c:119:vvp_thread_key_init()) slab-alloced 'vti': 584 at ffff988d03858b68. 02000000:00000001:2.0:1773941498.743506:0:14588:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.743510:0:2401:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:2.0:1773941498.743514:0:14588:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d0350f600. 00000080:00000010:3.0:1773941498.743516:0:14663:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff988d03858b68. 00000008:00000010:1.0:1773941498.743516:0:2401:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff988d08946f90. 00000100:00000001:2.0:1773941498.743519:0:14588:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:3.0:1773941498.743523:0:14663:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff988d054a8490. 00020000:00000010:1.0:1773941498.743523:0:2401:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff988d180f7e40. 00010000:00000001:2.0:1773941498.743524:0:14588:0:(ldlm_request.c:1348:ldlm_cancel_pack()) Process entered 00000080:00000010:3.0:1773941498.743529:0:14663:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff988d054a9da8. 00000008:00000010:1.0:1773941498.743532:0:2401:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff988d07158840. 00010000:00010000:2.0:1773941498.743533:0:14588:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: MGC192.168.204.114@tcp lock: ffff988d089dfc00/0x8e86767622e2895 lrc: 2/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5809400000020 nid: local remote: 0xd03f0241a6d949d3 expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000080:00000010:3.0:1773941498.743534:0:14663:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff988d054a96d0. 00020000:00000010:1.0:1773941498.743536:0:2401:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff988d182a8240. 00000080:00000010:3.0:1773941498.743539:0:14663:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff988d054ab6c0. 00000100:00000001:1.0:1773941498.743542:0:2401:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000080:00000010:3.0:1773941498.743544:0:14663:0:(vvp_dev.c:159:vvp_device_free()) kfreed 'vdv': 72 at ffff988d18350960. 00000100:00000001:1.0:1773941498.743546:0:2401:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00020000:00000001:3.0:1773941498.743551:0:14663:0:(lov_obd.c:754:lov_cleanup()) Process entered 00010000:00010000:2.0:1773941498.743552:0:14588:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00000100:00000040:1.0:1773941498.743555:0:2401:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c3480 x1860112291475712/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl New:QU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1773941498.743556:0:14588:0:(ldlm_request.c:1380:ldlm_cancel_pack()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941498.743561:0:14588:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941498.743566:0:14588:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941498.743571:0:14663:0:(lu_tgt_pool.c:211:lu_tgt_pool_free()) Process entered 00000100:00000001:1.0:1773941498.743573:0:2401:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000010:3.0:1773941498.743576:0:14663:0:(lu_tgt_pool.c:218:lu_tgt_pool_free()) kfreed 'op->op_array': 8 at ffff988d08b172c0. 00000100:00000040:2.0:1773941498.743578:0:14588:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000004a7e75f8] to pc [ptlrpcd_01_01+1] req@ffff988d06507100 x1860112291475840/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl New:QU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.743578:0:2401:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941498.743580:0:14663:0:(lu_tgt_pool.c:224:lu_tgt_pool_free()) Process leaving 02000000:00000001:1.0:1773941498.743587:0:2401:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941498.743592:0:2401:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941498.743597:0:2401:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941498.743603:0:2401:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:48257395-ca75-4e5f-a6ae-43b9fcf66fe6:2401:1860112291475712:192.168.204.114@tcp:103:ldlm_bl.0 00000100:00000001:1.0:1773941498.743608:0:2401:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00000001:2.0:1773941498.743613:0:14588:0:(ldlm_request.c:1464:ldlm_cli_cancel_req()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941498.743614:0:2401:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1773941498.743615:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1773941498.743619:0:2401:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941498.743620:0:14588:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941498.743621:0:2404:0:(client.c:1974:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1773941498.743624:0:2401:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1773941498.743625:0:2404:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00010000:00000001:2.0:1773941498.743625:0:14588:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941498.743629:0:14588:0:(ldlm_request.c:2290:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941498.743634:0:2404:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06507100 x1860112291475840/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl New:QU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1773941498.743635:0:14588:0:(ldlm_request.c:1689:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941498.743636:0:2401:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff988d08a4e800. 10000000:00000001:2.0:1773941498.743641:0:14588:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941498.743642:0:2401:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941498.743650:0:14588:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d089dfc00/0x8e86767622e2895 lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5809400000020 nid: local remote: 0xd03f0241a6d949d3 expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941498.743651:0:2404:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941498.743655:0:2404:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941498.743655:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 02000000:00000001:3.0:1773941498.743662:0:2404:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000400:00000010:1.0:1773941498.743663:0:2401:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1cd10. 02000000:00000001:3.0:1773941498.743665:0:2404:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00010000:00000001:2.0:1773941498.743668:0:14588:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 02000000:00000001:3.0:1773941498.743669:0:2404:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1773941498.743672:0:2401:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de550. 00010000:00010000:2.0:1773941498.743673:0:14588:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d089dfc00/0x8e86767622e2895 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x2:0x0].0x0 rrc: 2 type: PLN flags: 0x5809400000020 nid: local remote: 0xd03f0241a6d949d3 expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1773941498.743675:0:2404:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06507100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:48257395-ca75-4e5f-a6ae-43b9fcf66fe6:2404:1860112291475840:192.168.204.114@tcp:103:ldlm_bl.0 00000100:00000200:1.0:1773941498.743681:0:2401:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112291475712, portal 18 00000100:00000001:3.0:1773941498.743682:0:2404:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1773941498.743686:0:2401:0:(client.c:3244:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1773941498.743688:0:2404:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1773941498.743689:0:2401:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631468160 : -113743078083456 : ffff988d203c3480) 02000000:00000001:3.0:1773941498.743691:0:2404:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1773941498.743694:0:14588:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d071eec00 count: 0 02000000:00000001:3.0:1773941498.743695:0:2404:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:1.0:1773941498.743699:0:2401:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c3480 x1860112291475712/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941514 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1773941498.743706:0:2404:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff988d1817c600. 00010000:00000001:2.0:1773941498.743706:0:14588:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:1.0:1773941498.743708:0:2401:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1773941498.743711:0:2404:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941498.743711:0:14588:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000100:00000040:1.0:1773941498.743716:0:2401:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00000010:2.0:1773941498.743718:0:14588:0:(ldlm_lockd.c:2913:ldlm_bl_thread_blwi()) kfreed 'blwi': 192 at ffff988d1837d9c0. 00000400:00000010:1.0:1773941498.743719:0:2401:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de4c8. 00000400:00000200:3.0:1773941498.743721:0:2404:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000200:1.0:1773941498.743725:0:2401:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 17, xid 1860112291475712, offset 0 00000400:00000010:3.0:1773941498.743728:0:2404:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccddc0. 00010000:00000001:2.0:1773941498.743728:0:14588:0:(ldlm_lockd.c:2917:ldlm_bl_thread_blwi()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941498.743734:0:2401:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000020:00000001:2.0:1773941498.743735:0:14588:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000400:00000010:3.0:1773941498.743742:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac990. 00000400:00000200:1.0:1773941498.743744:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000020:00000001:2.0:1773941498.743747:0:14588:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1773941498.743749:0:2404:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112291475840, portal 18 00010000:00010000:2.0:1773941498.743752:0:14588:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1773941498.743753:0:2404:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000400:00000200:1.0:1773941498.743754:0:2401:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000020:00000001:2.0:1773941498.743755:0:14588:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000100:00000001:3.0:1773941498.743757:0:2404:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196586752 : -113743512964864 : ffff988d06507100) 00000020:00000001:2.0:1773941498.743760:0:14588:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941498.743762:0:2401:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00010000:2.0:1773941498.743765:0:14588:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1773941498.743767:0:2404:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06507100 x1860112291475840/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941514 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1773941498.743769:0:14588:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000400:00000200:1.0:1773941498.743771:0:2401:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000020:00000001:2.0:1773941498.743773:0:14588:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941498.743778:0:2401:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00010000:2.0:1773941498.743780:0:14588:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1773941498.743785:0:2404:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000400:00000200:1.0:1773941498.743786:0:2401:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000100:00000001:2.0:1773941498.743791:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1773941498.743792:0:2404:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941498.743795:0:2401:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [104:104:256:4294967295] 192.168.204.114@tcp LPNI seq info [104:104:8:4294967295] 00000400:00000010:3.0:1773941498.743797:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac5d8. 00000100:00000001:2.0:1773941498.743800:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1773941498.743803:0:2404:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 17, xid 1860112291475840, offset 0 00000400:00000200:1.0:1773941498.743803:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000020:00000001:2.0:1773941498.743807:0:14667:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941498.743811:0:14667:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941498.743813:0:2404:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00010000:2.0:1773941498.743818:0:14667:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1773941498.743822:0:14667:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000400:00000200:3.0:1773941498.743823:0:2404:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.743823:0:2401:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000020:00000001:2.0:1773941498.743825:0:14667:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941498.743831:0:14667:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000200:3.0:1773941498.743832:0:2404:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000800:00000200:1.0:1773941498.743837:0:2401:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941498.743838:0:2404:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000800:00000010:1.0:1773941498.743844:0:2401:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfd600. 00000400:00000200:3.0:1773941498.743845:0:2404:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941498.743850:0:2404:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941498.743853:0:2404:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000800:00000200:1.0:1773941498.743857:0:2401:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000400:00000200:3.0:1773941498.743863:0:2404:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [105:105:255:4294967295] 192.168.204.114@tcp LPNI seq info [105:105:7:4294967295] 00000800:00000200:1.0:1773941498.743870:0:2401:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000400:00000200:3.0:1773941498.743875:0:2404:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:1.0:1773941498.743877:0:2401:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfd600 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941498.743889:0:2401:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941498.743892:0:2404:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000400:00000200:1.0:1773941498.743896:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000800:00000200:3.0:1773941498.743905:0:2404:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.114@tcp 00000100:00000001:1.0:1773941498.743907:0:2401:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294890339 00020000:00020000:2.0:1773941498.743911:0:14663:0:(lov_obd.c:783:lov_cleanup()) lustre-clilov-ffff988d08f0a000: lov tgt 0 not cleaned! deathrow=0, lovrc=1 00000800:00000010:3.0:1773941498.743913:0:2404:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d1a2ec800. 00000100:00000010:1.0:1773941498.743914:0:2401:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3de0. 00000800:00000200:3.0:1773941498.743924:0:2404:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000100:00000001:1.0:1773941498.743924:0:2401:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1773941498.743932:0:2401:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294890339 00000800:00000200:3.0:1773941498.743936:0:2404:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000100:00000010:1.0:1773941498.743938:0:2401:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3f00. 00000800:00000200:3.0:1773941498.743940:0:2404:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d1a2ec800 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941498.743942:0:2401:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1773941498.743946:0:2404:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.743947:0:2401:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941498.743952:0:2401:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941498.743956:0:2404:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941498.743958:0:2401:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.743984:0:2401:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941498.743989:0:2401:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941498.743990:0:2404:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294890340 00000100:00000001:1.0:1773941498.743992:0:2401:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:1.0:1773941498.743997:0:2401:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941498.743998:0:2401:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941498.743999:0:2404:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294890339 00000100:00000001:1.0:1773941498.744000:0:2401:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941498.744005:0:2404:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294890340 00000100:00000001:1.0:1773941498.744006:0:2401:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941498.744014:0:2404:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294890339 00000100:00000001:3.0:1773941498.744018:0:2404:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941498.744020:0:2404:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941498.744023:0:2404:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941498.744024:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941498.744028:0:2404:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941498.744030:0:2404:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:3.0:1773941498.744033:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941498.744034:0:2404:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941498.744036:0:2404:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941498.744038:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941498.744051:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941498.744055:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfd600. 00000400:00000200:1.0:1773941498.744059:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941498.744065:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941498.744069:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de4c8 00000400:00000010:1.0:1773941498.744071:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de4c8. 00000100:00000001:1.0:1773941498.744076:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941498.744081:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c3480 x1860112291475712/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941514 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.744089:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941498.744093:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c3480 x1860112291475712/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941514 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.744098:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.744100:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000200:1.0:1773941498.744162:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941498.744164:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d1a2ec800. 00000400:00000200:1.0:1773941498.744167:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941498.744170:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941498.744178:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac5d8 00000400:00000010:1.0:1773941498.744179:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac5d8. 00000100:00000001:1.0:1773941498.744182:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941498.744186:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06507100 x1860112291475840/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941514 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.744191:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941498.744194:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06507100 x1860112291475840/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941514 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.744198:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.744199:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000008:00000010:1.0:1773941498.744211:0:2402:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff988d071598c0. 00020000:00000010:1.0:1773941498.744227:0:2402:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff988d182aaac0. 00000100:00000001:1.0:1773941498.744230:0:2402:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941498.744231:0:2402:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941498.744235:0:2402:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c2a00 x1860112291475584/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl New:QU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.744241:0:2402:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941498.744242:0:2402:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941498.744244:0:2402:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941498.744245:0:2402:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941498.744246:0:2402:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941498.744249:0:2402:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:48257395-ca75-4e5f-a6ae-43b9fcf66fe6:2402:1860112291475584:192.168.204.114@tcp:103:ldlm_bl.0 00000100:00000001:1.0:1773941498.744251:0:2402:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941498.744253:0:2402:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941498.744255:0:2402:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941498.744256:0:2402:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941498.744259:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff988d08a4ea00. 02000000:00000001:1.0:1773941498.744261:0:2402:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941498.744264:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941498.744267:0:2402:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c580. 00000400:00000010:1.0:1773941498.744269:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de4c8. 00000100:00000200:1.0:1773941498.744272:0:2402:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112291475584, portal 18 00000100:00000001:1.0:1773941498.744273:0:2402:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941498.744280:0:2402:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631465472 : -113743078086144 : ffff988d203c2a00) 00000100:00000040:1.0:1773941498.744283:0:2402:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c2a00 x1860112291475584/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941514 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.744288:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941498.744290:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941498.744292:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de440. 00000100:00000200:1.0:1773941498.744294:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 17, xid 1860112291475584, offset 0 00000400:00000200:1.0:1773941498.744298:0:2402:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941498.744301:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.744304:0:2402:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941498.744307:0:2402:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941498.744309:0:2402:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941498.744311:0:2402:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941498.744313:0:2402:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:1.0:1773941498.744316:0:2402:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [106:106:256:4294967295] 192.168.204.114@tcp LPNI seq info [106:106:8:4294967295] 00000400:00000200:1.0:1773941498.744319:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.744324:0:2402:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:1.0:1773941498.744327:0:2402:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:1.0:1773941498.744329:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfd600. 00000800:00000200:1.0:1773941498.744332:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:1.0:1773941498.744335:0:2402:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:1.0:1773941498.744338:0:2402:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfd600 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941498.744345:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941498.744347:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941498.744348:0:2402:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294890340 00000100:00000001:1.0:1773941498.744351:0:2402:0:(niobuf.c:691:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, keep current deadline 4294890340 00000100:00000001:1.0:1773941498.744352:0:2402:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294890340 00000100:00000001:1.0:1773941498.744354:0:2402:0:(niobuf.c:691:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, keep current deadline 4294890340 00000100:00000001:1.0:1773941498.744356:0:2402:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941498.744357:0:2402:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.744364:0:2402:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.744365:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941498.744368:0:2402:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941498.744369:0:2402:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:1.0:1773941498.744371:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941498.744372:0:2402:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941498.744374:0:2402:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.744375:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941498.744416:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941498.744420:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfd600. 00000400:00000200:1.0:1773941498.744424:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941498.744428:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941498.744432:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de440 00000400:00000010:1.0:1773941498.744434:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de440. 00000100:00000001:1.0:1773941498.744437:0:2394:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941498.744442:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c2a00 x1860112291475584/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941514 ref 2 fl Rpc:r/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.744450:0:2394:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941498.744454:0:2394:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c2a00 x1860112291475584/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941514 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.744461:0:2394:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.744463:0:2394:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:1.0:1773941498.746162:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941498.746172:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941498.746175:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941498.746178:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941498.746183:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.746188:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941498.746192:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.746195:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.746198:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.746201:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 18 MB=0x69bc324603180 00000400:00000200:1.0:1773941498.746204:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.746208:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 12 from 12345-192.168.204.114@tcp of length 224/224 into md 0x879 [1] + 224 00000800:00000001:1.0:1773941498.746213:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941498.746237:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941498.746239:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941498.746243:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:1.0:1773941498.746246:0:2393:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941498.746250:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06507100 x1860112291475840/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941514 ref 1 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941498.746259:0:2393:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff988d06507100 x1860112291475840/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941514 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.746270:0:2393:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1773941498.746276:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941498.746280:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941498.746282:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941498.746284:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941498.746287:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.746291:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941498.746295:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.746297:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.746299:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.746302:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 18 MB=0x69bc324603100 00000400:00000200:1.0:1773941498.746305:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.746308:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 12 from 12345-192.168.204.114@tcp of length 224/224 into md 0x869 [1] + 224 00000800:00000001:1.0:1773941498.746310:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941498.746315:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941498.746316:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941498.746318:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:1.0:1773941498.746320:0:2393:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941498.746322:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c3480 x1860112291475712/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941514 ref 1 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941498.746328:0:2393:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff988d203c3480 x1860112291475712/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941514 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.746335:0:2393:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1773941498.746343:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941498.746345:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941498.746351:0:2401:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941498.746353:0:2401:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941498.746357:0:2401:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941498.746358:0:2401:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941498.746360:0:2401:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1cd10. 00000400:00000200:1.0:1773941498.746363:0:2401:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de550 00000400:00000010:1.0:1773941498.746365:0:2401:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de550. 00000100:00000001:1.0:1773941498.746367:0:2401:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941498.746370:0:2401:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c3480 x1860112291475712/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941514 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941498.746377:0:2401:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c3480 x1860112291475712/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941514 ref 1 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.746381:0:2401:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941498.746383:0:2401:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941498.746384:0:2401:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941498.746386:0:2401:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941498.746388:0:2401:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941498.746390:0:2401:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941498.746392:0:2401:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941498.746398:0:2401:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d035c4ca0 time=1773941498 v=5 (0 0 0 0) 00000100:00001000:1.0:1773941498.746403:0:2401:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d035c4c10 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941498.746406:0:2401:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1773941498.746408:0:2401:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941498.746410:0:2401:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:1.0:1773941498.746413:0:2401:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.746415:0:2401:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941498.746421:0:2401:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c3480 x1860112291475712/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941514 ref 1 fl Rpc:RQU/200/0 rc 0/0 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.746429:0:2401:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941498.746433:0:2401:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941498.746435:0:2401:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941498.746441:0:2401:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c3480 x1860112291475712/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941514 ref 1 fl Interpret:RQU/200/0 rc 0/0 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941498.746464:0:2401:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:48257395-ca75-4e5f-a6ae-43b9fcf66fe6:2401:1860112291475712:192.168.204.114@tcp:103:ldlm_bl.0 00000100:00000001:1.0:1773941498.746469:0:2401:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941498.746472:0:2401:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941498.746476:0:2401:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c3480 x1860112291475712/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941514 ref 1 fl Complete:RQU/200/0 rc 0/0 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.746481:0:2401:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941498.746484:0:2401:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941498.746486:0:2401:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff988d08a4e800. 02000000:00000001:1.0:1773941498.746489:0:2401:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941498.746490:0:2401:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941498.746492:0:2401:0:(genops.c:1242:class_import_put()) import ffff988d035c4800 refcount=5 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941498.746494:0:2401:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941498.746496:0:2401:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d08a4f800. 02000000:00000001:1.0:1773941498.746499:0:2401:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941498.746500:0:2401:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941498.746502:0:2401:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c3480. 00000100:00000001:1.0:1773941498.746505:0:2401:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941498.746506:0:2401:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941498.746507:0:2401:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941498.746511:0:2401:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941498.746513:0:2401:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.746515:0:2401:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941498.746517:0:2401:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:1.0:1773941498.746528:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000100:00000001:3.0:1773941498.746530:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000001:1.0:1773941498.746533:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941498.746534:0:2404:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000800:00000001:1.0:1773941498.746538:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000400:00000001:3.0:1773941498.746539:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000800:00000001:1.0:1773941498.746541:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:3.0:1773941498.746543:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1773941498.746546:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941498.746547:0:2404:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccddc0. 00000400:00000200:3.0:1773941498.746554:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac990 00000400:00000200:1.0:1773941498.746555:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000010:3.0:1773941498.746556:0:2404:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac990. 00000100:00000001:3.0:1773941498.746561:0:2404:0:(events.c:78:reply_in_callback()) Process entered 00000400:00000200:1.0:1773941498.746565:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000200:3.0:1773941498.746567:0:2404:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06507100 x1860112291475840/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941514 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1773941498.746574:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.746585:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.746591:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 18 MB=0x69bc324603080 00000400:00000200:1.0:1773941498.746603:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.746607:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 12 from 12345-192.168.204.114@tcp of length 224/224 into md 0x881 [1] + 224 00000800:00000001:1.0:1773941498.746613:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941498.746642:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941498.746644:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941498.746648:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000200:3.0:1773941498.746649:0:2404:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06507100 x1860112291475840/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941514 ref 1 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.746652:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1773941498.746660:0:2404:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000200:1.0:1773941498.746661:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c2a00 x1860112291475584/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941514 ref 1 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941498.746663:0:2404:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941498.746668:0:2404:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941498.746671:0:2404:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941498.746673:0:2404:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941498.746676:0:2404:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941498.746682:0:2404:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941498.746686:0:2404:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d035c4ca0 time=0 v=5 (5 0 0 0) 00000100:00000040:1.0:1773941498.746687:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff988d203c2a00 x1860112291475584/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941514 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00001000:3.0:1773941498.746689:0:2404:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d035c4c10 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941498.746702:0:2404:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1773941498.746709:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1773941498.746717:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941498.746720:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941498.746731:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941498.746734:0:2402:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941498.746737:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941498.746739:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941498.746742:0:2402:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c580. 00000400:00000200:1.0:1773941498.746745:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de4c8 00000400:00000010:1.0:1773941498.746748:0:2402:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de4c8. 00000100:00000001:1.0:1773941498.746751:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941498.746756:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c2a00 x1860112291475584/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941514 ref 1 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941498.746767:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c2a00 x1860112291475584/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941514 ref 1 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941498.746769:0:2404:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941498.746774:0:2404:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941498.746777:0:2404:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941498.746782:0:2404:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.746784:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941498.746788:0:2402:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941498.746789:0:2404:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06507100 x1860112291475840/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941514 ref 1 fl Rpc:RQU/200/0 rc 0/0 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.746793:0:2402:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941498.746798:0:2402:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941498.746800:0:2404:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941498.746801:0:2402:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941498.746805:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941498.746806:0:2402:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941498.746808:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1773941498.746809:0:2402:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941498.746813:0:2404:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06507100 x1860112291475840/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941514 ref 1 fl Interpret:RQU/200/0 rc 0/0 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00001000:1.0:1773941498.746822:0:2402:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d035c4ca0 time=0 v=5 (5 0 0 0) 00000100:00100000:3.0:1773941498.746824:0:2404:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06507100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:48257395-ca75-4e5f-a6ae-43b9fcf66fe6:2404:1860112291475840:192.168.204.114@tcp:103:ldlm_bl.0 00000100:00001000:1.0:1773941498.746828:0:2402:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d035c4c10 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941498.746831:0:2404:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941498.746832:0:2402:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941498.746835:0:2404:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1773941498.746836:0:2402:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941498.746839:0:2402:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00000100:00000040:3.0:1773941498.746842:0:2404:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06507100 x1860112291475840/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941514 ref 1 fl Complete:RQU/200/0 rc 0/0 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941498.746843:0:2402:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.746847:0:2402:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941498.746854:0:2404:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000040:1.0:1773941498.746856:0:2402:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c2a00 x1860112291475584/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941514 ref 1 fl Rpc:RQU/200/0 rc 0/0 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1773941498.746857:0:2404:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941498.746862:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff988d1817c600. 00000100:00000001:1.0:1773941498.746864:0:2402:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941498.746868:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1773941498.746869:0:2404:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:1.0:1773941498.746870:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1773941498.746873:0:2404:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000040:1.0:1773941498.746875:0:2402:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c2a00 x1860112291475584/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941514 ref 1 fl Interpret:RQU/200/0 rc 0/0 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1773941498.746878:0:2404:0:(genops.c:1242:class_import_put()) import ffff988d035c4800 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941498.746884:0:2404:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941498.746887:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d0350f600. 00000100:00100000:1.0:1773941498.746888:0:2402:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:48257395-ca75-4e5f-a6ae-43b9fcf66fe6:2402:1860112291475584:192.168.204.114@tcp:103:ldlm_bl.0 02000000:00000001:3.0:1773941498.746895:0:2404:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1773941498.746896:0:2402:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1773941498.746900:0:2404:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1773941498.746900:0:2402:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000010:3.0:1773941498.746905:0:2404:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06507100. 00000100:00000040:1.0:1773941498.746906:0:2402:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c2a00 x1860112291475584/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 1773941514 ref 1 fl Complete:RQU/200/0 rc 0/0 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941498.746913:0:2404:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941498.746914:0:2402:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1773941498.746918:0:2404:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1773941498.746918:0:2402:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941498.746921:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff988d08a4ea00. 00000100:00000001:3.0:1773941498.746923:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1773941498.746925:0:2402:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941498.746928:0:2402:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941498.746929:0:2402:0:(genops.c:1242:class_import_put()) import ffff988d035c4800 refcount=3 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941498.746930:0:2404:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:1.0:1773941498.746931:0:2402:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941498.746935:0:2404:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941498.746935:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d08a4e200. 02000000:00000001:1.0:1773941498.746938:0:2402:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941498.746939:0:2402:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941498.746940:0:2402:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c2a00. 00000100:00000001:1.0:1773941498.746943:0:2402:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941498.746946:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941498.746946:0:2402:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941498.746947:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941498.746950:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.746958:0:2402:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941498.746979:0:2402:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.746981:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941498.746983:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1773941498.749043:0:14663:0:(lov_obd.c:562:lov_del_target()) Process entered 00020000:01000000:2.0:1773941498.749188:0:14663:0:(lov_obd.c:589:lov_del_target()) uuid: lustre-OST0000_UUID idx: 0 gen: 1 exp: ffff988d035c2000 active: 1 00020000:00000001:2.0:1773941498.749195:0:14663:0:(lov_obd.c:598:lov_del_target()) Process leaving (rc=0 : 0 : 0) 00020000:01000000:2.0:1773941498.749199:0:14663:0:(lov_obd.c:64:lov_tgts_putref()) destroying 1 lov targets 00000020:00000001:2.0:1773941498.749202:0:14663:0:(lu_tgt_pool.c:163:lu_tgt_pool_remove()) Process entered 00000020:00000001:2.0:1773941498.749205:0:14663:0:(lu_tgt_pool.c:180:lu_tgt_pool_remove()) Process leaving (rc=18446744073709551594 : -22 : ffffffffffffffea) 00020000:00000001:2.0:1773941498.749209:0:14663:0:(lu_tgt_descs.c:164:lu_qos_del_tgt()) Process entered 00020000:00000001:2.0:1773941498.749212:0:14663:0:(lu_tgt_descs.c:169:lu_qos_del_tgt()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00020000:00000001:2.0:1773941498.749231:0:14663:0:(lu_tgt_descs.c:186:lu_qos_del_tgt()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00020000:01000000:2.0:1773941498.749235:0:14663:0:(lov_obd.c:611:__lov_del_obd()) Removing tgt lustre-OST0000_UUID : lustre-OST0000-osc-ffff988d08f0a000 00020000:00000001:2.0:1773941498.749238:0:14663:0:(lov_obd.c:245:lov_disconnect_obd()) Process entered 00020000:01000000:2.0:1773941498.749240:0:14663:0:(lov_obd.c:248:lov_disconnect_obd()) lustre-clilov-ffff988d08f0a000: disconnecting target lustre-OST0000-osc-ffff988d08f0a000 00020000:00000001:2.0:1773941498.749242:0:14663:0:(obd_class.h:1347:obd_register_observer()) Process entered 00020000:00000001:2.0:1773941498.749245:0:14663:0:(obd_class.h:1360:obd_register_observer()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1773941498.749247:0:14663:0:(obd_class.h:902:obd_disconnect()) Process entered 00010000:00000001:2.0:1773941498.749250:0:14663:0:(ldlm_lib.c:759:client_disconnect_export()) Process entered 00010000:00000040:2.0:1773941498.749253:0:14663:0:(ldlm_lib.c:772:client_disconnect_export()) disconnect lustre-OST0000-osc-ffff988d08f0a000 - 1 00000100:00000001:2.0:1773941498.749256:0:14663:0:(pinger.c:388:ptlrpc_pinger_del_import()) Process entered 00000100:00080000:2.0:1773941498.749264:0:14663:0:(pinger.c:396:ptlrpc_pinger_del_import()) removing pingable import e9ee4b2c-2bac-44e1-850a-8bf1479379ac->lustre-OST0000_UUID 00000020:00000001:2.0:1773941498.749267:0:14663:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941498.749269:0:14663:0:(genops.c:1242:class_import_put()) import ffff988d180dc800 refcount=1 obd=lustre-OST0000-osc-ffff988d08f0a000 00000020:00000001:2.0:1773941498.749272:0:14663:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941498.749274:0:14663:0:(pinger.c:401:ptlrpc_pinger_del_import()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941498.749277:0:14663:0:(ldlm_request.c:2367:ldlm_cli_cancel_unused()) Process entered 00000001:00000001:2.0:1773941498.749280:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941498.749283:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941498.749326:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941498.749329:0:14663:0:(ldlm_request.c:2379:ldlm_cli_cancel_unused()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941498.749332:0:14663:0:(ldlm_resource.c:1268:ldlm_namespace_free_prior()) Process entered 00010000:00000001:2.0:1773941498.749334:0:14663:0:(ldlm_resource.c:1207:__ldlm_namespace_free()) Process entered 00000001:00000001:2.0:1773941498.749336:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941498.749337:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941498.749369:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:2.0:1773941498.749371:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941498.749372:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941498.749404:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941498.749407:0:14663:0:(ldlm_resource.c:1251:__ldlm_namespace_free()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941498.749409:0:14663:0:(ldlm_resource.c:1293:ldlm_namespace_free_prior()) Process leaving 00000100:00000001:2.0:1773941498.749411:0:14663:0:(import.c:1924:ptlrpc_disconnect_import()) Process entered 00000100:00000001:2.0:1773941498.749415:0:14663:0:(import.c:1858:ptlrpc_disconnect_import_async()) Process entered 00000100:00000001:2.0:1773941498.749417:0:14663:0:(import.c:1750:ptlrpc_disconnect_prep_req()) Process entered 00000100:00000010:2.0:1773941498.749423:0:14663:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06504380. 00000020:00000040:2.0:1773941498.749427:0:14663:0:(genops.c:1229:class_import_get()) import ffff988d180dc800 refcount=2 obd=lustre-OST0000-osc-ffff988d08f0a000 00000100:00000001:2.0:1773941498.749431:0:14663:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941498.749433:0:14663:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941498.749437:0:14663:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941498.749442:0:14663:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff988d217a8e00. 00000100:00000001:2.0:1773941498.749447:0:14663:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.749450:0:14663:0:(import.c:1788:ptlrpc_disconnect_prep_req()) Process leaving (rc=18446630330196575104 : -113743512976512 : ffff988d06504380) 00000100:00080000:2.0:1773941498.749454:0:14663:0:(import.c:65:import_set_state_nolock()) ffff988d180dc800 lustre-OST0000_UUID: changing import state from FULL to CONNECTING 00000100:00000001:2.0:1773941498.749458:0:14663:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941498.749462:0:14663:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941498.749470:0:14663:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000ee6d5628] to pc [ptlrpcd_rcv+-1] req@ffff988d06504380 x1860112291475968/t0(0) o9->lustre-OST0000-osc-ffff988d08f0a000@192.168.204.114@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941498.749504:0:14663:0:(import.c:1904:ptlrpc_disconnect_import_async()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.749524:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941498.749531:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941498.749533:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941498.749539:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06504380 x1860112291475968/t0(0) o9->lustre-OST0000-osc-ffff988d08f0a000@192.168.204.114@tcp:28/4 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.749545:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941498.749547:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941498.749549:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941498.749551:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941498.749553:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941498.749556:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06504380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9ee4b2c-2bac-44e1-850a-8bf1479379ac:2400:1860112291475968:192.168.204.114@tcp:9:umount.0 00000100:00000001:1.0:1773941498.749560:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941498.749564:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941498.749566:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941498.749568:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941498.749574:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff988d08a4e200. 02000000:00000001:1.0:1773941498.749576:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941498.749581:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941498.749585:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c580. 00000400:00000010:1.0:1773941498.749590:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de4c8. 00000100:00000200:1.0:1773941498.749595:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112291475968, portal 4 00000100:00000001:1.0:1773941498.749597:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941498.749599:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196575104 : -113743512976512 : ffff988d06504380) 00000100:00000040:1.0:1773941498.749606:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06504380 x1860112291475968/t0(0) o9->lustre-OST0000-osc-ffff988d08f0a000@192.168.204.114@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941508 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.749614:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941498.749617:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941498.749621:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de550. 00000100:00000200:1.0:1773941498.749624:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 28, xid 1860112291475968, offset 0 00000400:00000200:1.0:1773941498.749630:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941498.749643:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.749649:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941498.749653:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941498.749657:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941498.749660:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941498.749663:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:1.0:1773941498.749667:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [107:107:256:4294967295] 192.168.204.114@tcp LPNI seq info [107:107:8:4294967295] 00000400:00000200:1.0:1773941498.749672:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.749682:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:1.0:1773941498.749687:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:1.0:1773941498.749692:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfd600. 00000800:00000200:1.0:1773941498.749698:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:1.0:1773941498.749703:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:1.0:1773941498.749706:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfd600 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1773941498.749719:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941498.749722:0:2400:0:(niobuf.c:634:kick_cpu_latency()) lustre-OST0000-osc-ffff988d08f0a000: using a timeout of 3040 usecs (4 jiffies) 00000400:00000200:1.0:1773941498.749725:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941498.749728:0:2400:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff988d08f0a000: PM QoS new deadline estimation for cpu 0 is 4294889094 00000100:00000001:1.0:1773941498.749730:0:2400:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff988d08f0a000: PM QoS request active for cpu 0, keep current deadline 4294890340 00000100:00000001:1.0:1773941498.749732:0:2400:0:(niobuf.c:662:kick_cpu_latency()) lustre-OST0000-osc-ffff988d08f0a000: PM QoS new deadline estimation for cpu 1 is 4294889094 00000100:00000001:1.0:1773941498.749734:0:2400:0:(niobuf.c:691:kick_cpu_latency()) lustre-OST0000-osc-ffff988d08f0a000: PM QoS request active for cpu 1, keep current deadline 4294890340 00000100:00000001:1.0:1773941498.749735:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941498.749737:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.749740:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.749743:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941498.749748:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941498.749749:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1773941498.749752:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941498.749754:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941498.749756:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.749759:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941498.749807:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941498.749813:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfd600. 00000400:00000200:0.0:1773941498.749819:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941498.749824:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941498.749829:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de550 00000400:00000010:0.0:1773941498.749832:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de550. 00000100:00000001:0.0:1773941498.749836:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941498.749841:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06504380 x1860112291475968/t0(0) o9->lustre-OST0000-osc-ffff988d08f0a000@192.168.204.114@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941508 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941498.749852:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941498.749857:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06504380 x1860112291475968/t0(0) o9->lustre-OST0000-osc-ffff988d08f0a000@192.168.204.114@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941508 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941498.749865:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.749867:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000080:00000010:3.1:1773941498.750287:0:14669:0:(dcache.c:31:free_dentry_data()) kfreed 'lld': 24 at ffff988d111d6c40. 00000080:00000010:3.1:1773941498.750300:0:14669:0:(super25.c:59:ll_inode_destroy_callback()) slab-freed '(ptr)': 2072 at ffff988d205298c0. 00000800:00000001:1.0:1773941498.751522:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941498.751531:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941498.751534:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941498.751537:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941498.751541:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.751546:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941498.751551:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.751554:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.751558:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.751560:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 4 MB=0x69bc324603200 00000400:00000200:1.0:1773941498.751564:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.751567:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 4 from 12345-192.168.204.114@tcp of length 224/224 into md 0x891 [1] + 224 00000800:00000001:1.0:1773941498.751572:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941498.751579:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941498.751580:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941498.751584:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:1.0:1773941498.751588:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941498.751594:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06504380 x1860112291475968/t0(0) o9->lustre-OST0000-osc-ffff988d08f0a000@192.168.204.114@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941508 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941498.751605:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff988d06504380 x1860112291475968/t0(0) o9->lustre-OST0000-osc-ffff988d08f0a000@192.168.204.114@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941508 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.751624:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1773941498.751631:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941498.751634:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941498.751652:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941498.751656:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941498.751661:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941498.751663:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941498.751666:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c580. 00000400:00000200:0.0:1773941498.751671:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de4c8 00000400:00000010:0.0:1773941498.751673:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de4c8. 00000100:00000001:0.0:1773941498.751676:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941498.751682:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06504380 x1860112291475968/t0(0) o9->lustre-OST0000-osc-ffff988d08f0a000@192.168.204.114@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941508 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941498.751694:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06504380 x1860112291475968/t0(0) o9->lustre-OST0000-osc-ffff988d08f0a000@192.168.204.114@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941508 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941498.751702:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941498.751704:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941498.751707:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941498.751711:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941498.751720:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941498.751723:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941498.751726:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941498.751733:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d180dcc58 time=1 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941498.751738:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d180dcc10 time=1 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941498.751742:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941498.751744:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941498.751747:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00080000:0.0:1773941498.751753:0:2400:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff988d06504380 x1860112291475968/t0(0) o9->lustre-OST0000-osc-ffff988d08f0a000@192.168.204.114@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941508 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'umount.0' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1773941498.751763:0:2400:0:(ldlm_request.c:1535:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.751766:0:2400:0:(client.c:3065:ptlrpc_free_committed()) Process entered 00000100:00000040:0.0:1773941498.751769:0:2400:0:(client.c:3077:ptlrpc_free_committed()) lustre-OST0000-osc-ffff988d08f0a000: skip recheck: last_committed 0 00000100:00000001:0.0:1773941498.751771:0:2400:0:(client.c:3078:ptlrpc_free_committed()) Process leaving 00000100:00000001:0.0:1773941498.751774:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941498.751779:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06504380 x1860112291475968/t0(0) o9->lustre-OST0000-osc-ffff988d08f0a000@192.168.204.114@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941508 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941498.751788:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941498.751790:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941498.751792:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1773941498.751795:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d180dc800 lustre-OST0000_UUID: changing import state from CONNECTING to CLOSED 00000100:00000001:0.0:1773941498.751798:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 00000008:00000001:0.0:1773941498.751802:0:2400:0:(osc_request.c:3880:osc_import_event()) Process entered 00000008:00000001:0.0:1773941498.751804:0:2400:0:(osc_request.c:3947:osc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.751806:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941498.751808:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 00000008:00000001:0.0:1773941498.751809:0:2400:0:(osc_request.c:3880:osc_import_event()) Process entered 00000008:00000001:0.0:1773941498.751811:0:2400:0:(osc_request.c:3947:osc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.751813:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000040:0.0:1773941498.751827:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06504380 x1860112291475968/t0(0) o9->lustre-OST0000-osc-ffff988d08f0a000@192.168.204.114@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941508 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941498.751837:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06504380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9ee4b2c-2bac-44e1-850a-8bf1479379ac:2400:1860112291475968:192.168.204.114@tcp:9:umount.0 00000100:00000001:0.0:1773941498.751843:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941498.751851:0:14663:0:(import.c:1967:ptlrpc_disconnect_import()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.751854:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941498.751857:0:14663:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941498.751862:0:14663:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-OST0000_UUID INVALID 00000100:00000040:0.0:1773941498.751862:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06504380 x1860112291475968/t0(0) o9->lustre-OST0000-osc-ffff988d08f0a000@192.168.204.114@tcp:28/4 lens 224/224 e 0 to 0 dl 1773941508 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941498.751866:0:14663:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00000001:2.0:1773941498.751870:0:14663:0:(client.c:3065:ptlrpc_free_committed()) Process entered 00000100:00100000:2.0:1773941498.751874:0:14663:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0000-osc-ffff988d08f0a000: committing for last_committed 0 gen 2 00000100:00000001:0.0:1773941498.751878:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000040:2.0:1773941498.751880:0:14663:0:(client.c:3191:ptlrpc_free_committed()) lustre-OST0000-osc-ffff988d08f0a000: normal: skip=0 replay=0/0 committed=0/0 00000100:00000001:2.0:1773941498.751884:0:14663:0:(client.c:3192:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1773941498.751885:0:14663:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941498.751887:0:14663:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941498.751889:0:14663:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:0.0:1773941498.751890:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000008:00000001:2.0:1773941498.751891:0:14663:0:(osc_request.c:3880:osc_import_event()) Process entered 00000008:00000001:2.0:1773941498.751895:0:14663:0:(osc_request.c:3947:osc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941498.751896:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff988d08a4e200. 00000100:00000001:2.0:1773941498.751898:0:14663:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00100000:2.0:1773941498.751903:0:14663:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000001:0.0:1773941498.751904:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1773941498.751906:0:14663:0:(obd_class.h:1215:obd_import_event()) Process entered 00000020:00000001:0.0:1773941498.751908:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000008:00000001:2.0:1773941498.751909:0:14663:0:(osc_request.c:3880:osc_import_event()) Process entered 00000020:00000040:0.0:1773941498.751912:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d180dc800 refcount=1 obd=lustre-OST0000-osc-ffff988d08f0a000 00000001:00000001:2.0:1773941498.751914:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941498.751918:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000001:0.0:1773941498.751918:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1773941498.751924:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff988d217a8e00. 02000000:00000001:0.0:1773941498.751930:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1773941498.751932:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1773941498.751934:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06504380. 00000100:00000001:0.0:1773941498.751939:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941498.751941:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941498.751943:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941498.751948:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941498.751951:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.751954:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941498.751956:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:2.0:1773941498.751990:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:2.0:1773941498.751993:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941498.751996:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941498.752029:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941498.752033:0:14663:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1773941498.752053:0:14663:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff988d05490240. 00000020:00000010:2.0:1773941498.752060:0:14663:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d0350f000. 00000020:00000010:2.0:1773941498.752065:0:14663:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d0350fa00. 00000020:00000010:2.0:1773941498.752070:0:14663:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d08bbbb40. 00000008:00000010:2.0:1773941498.752075:0:14663:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff988d18285940. 00020000:00000010:2.0:1773941498.752080:0:14663:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff988d035a1300. 00008000:00000010:2.0:1773941498.752085:0:14663:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d08324710. 00000020:00000010:2.0:1773941498.752090:0:14663:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d0350ec00. 00000008:00000010:2.0:1773941498.752094:0:14663:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff988d03544420. 00020000:00000010:2.0:1773941498.752105:0:14663:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff988d02ca0000. 00008000:00000010:2.0:1773941498.752108:0:14663:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff988d111ef598. 00000020:00000001:2.0:1773941498.752111:0:14663:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446630330179322440 : -113743530229176 : ffff988d05490248) 00000020:00001000:2.0:1773941498.752114:0:14663:0:(cl_object.c:831:cl_env_get()) 1@ffff988d05490240 00000008:00000001:2.0:1773941498.752118:0:14663:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:2.0:1773941498.752120:0:14663:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:2.0:1773941498.752123:0:14663:0:(osc_cache.c:2316:osc_next_obj()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1773941498.752126:0:14663:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000001:00000001:2.0:1773941498.752128:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941498.752129:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941498.752164:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1773941498.752166:0:14663:0:(cl_object.c:925:cl_env_put()) 1@ffff988d05490240 00000020:00000010:2.0:1773941498.752170:0:14663:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d0350fa00. 00000020:00000010:2.0:1773941498.752175:0:14663:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d08bbbb40. 00000008:00000010:2.0:1773941498.752180:0:14663:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff988d18285940. 00020000:00000010:2.0:1773941498.752186:0:14663:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff988d035a1300. 00008000:00000010:2.0:1773941498.752190:0:14663:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d08324710. 00000020:00000010:2.0:1773941498.752195:0:14663:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0350f000. 00000008:00000010:2.0:1773941498.752198:0:14663:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff988d03544420. 00020000:00000010:2.0:1773941498.752201:0:14663:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff988d02ca0000. 00008000:00000010:2.0:1773941498.752205:0:14663:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff988d111ef598. 00000020:00000010:2.0:1773941498.752209:0:14663:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0350ec00. 00000020:00000010:2.0:1773941498.752212:0:14663:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff988d05490240. 00000001:00000001:2.0:1773941498.752226:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941498.752228:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941498.752276:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:2.0:1773941498.752279:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941498.752280:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941498.752327:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1773941498.752330:0:14663:0:(osc_request.c:3947:osc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.752332:0:14663:0:(obd_class.h:1226:obd_import_event()) Process leaving 00010000:00000001:2.0:1773941498.752336:0:14663:0:(ldlm_lib.c:818:client_disconnect_export()) Process leaving 00000020:00000001:2.0:1773941498.752339:0:14663:0:(genops.c:1462:class_disconnect()) Process entered 00000020:00000080:2.0:1773941498.752342:0:14663:0:(genops.c:1487:class_disconnect()) disconnect: cookie 0x8e86767622e28bf 00000020:00000040:2.0:1773941498.752348:0:14663:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d035c2000 with handle 0x8e86767622e28bf from hash 00000020:00000040:2.0:1773941498.752362:0:14663:0:(genops.c:898:class_export_put()) PUTting export ffff988d035c2000 : new refcount 2 00000020:00000001:2.0:1773941498.752377:0:14663:0:(genops.c:1934:obd_stale_export_put()) Process entered 00000020:00000040:2.0:1773941498.752379:0:14663:0:(genops.c:898:class_export_put()) PUTting export ffff988d035c2000 : new refcount 1 00000020:00000001:2.0:1773941498.752381:0:14663:0:(genops.c:1959:obd_stale_export_put()) Process leaving 00000020:00000040:2.0:1773941498.752383:0:14663:0:(genops.c:898:class_export_put()) PUTting export ffff988d035c2000 : new refcount 0 00000020:00000080:2.0:1773941498.752386:0:14663:0:(genops.c:904:class_export_put()) final put ffff988d035c2000/LOV_OSC_UUID 00000020:00000001:2.0:1773941498.752388:0:14663:0:(lprocfs_status_server.c:686:lprocfs_exp_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941498.752409:0:14663:0:(genops.c:1493:class_disconnect()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941498.752412:0:14663:0:(ldlm_lib.c:831:client_disconnect_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941498.752435:0:8:0:(genops.c:857:class_export_destroy()) Process entered 00000020:00000080:0.0:1773941498.752439:0:8:0:(genops.c:863:class_export_destroy()) destroying export ffff988d035c2000/LOV_OSC_UUID for lustre-OST0000-osc-ffff988d08f0a000 00000020:00000040:0.0:1773941498.752443:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 11 to 192.168.204.114@tcp 00000020:00000001:0.0:1773941498.752445:0:8:0:(obd_class.h:1017:obd_destroy_export()) Process entered 00000020:00000001:0.0:1773941498.752447:0:8:0:(obd_class.h:1021:obd_destroy_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1773941498.752450:0:8:0:(obd_config.c:1058:class_decref()) Decref lustre-OST0000-osc-ffff988d08f0a000 (ffff988d06460000) now 5 - export 00000020:00000010:0.0:1773941498.752453:0:8:0:(genops.c:879:class_export_destroy()) kfreed_rcu 'exp': 1208 at ffff988d035c2000. 00000020:00000001:0.0:1773941498.752458:0:8:0:(genops.c:880:class_export_destroy()) Process leaving 00020000:00000001:2.0:1773941498.752472:0:14663:0:(obd_class.h:914:obd_disconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1773941498.752476:0:14663:0:(lov_obd.c:280:lov_disconnect_obd()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:2.0:1773941498.752479:0:14663:0:(lov_obd.c:616:__lov_del_obd()) kfreed 'tgt': 368 at ffff988d065dd000. 00000020:00000001:2.0:1773941498.752485:0:14663:0:(obd_config.c:2489:class_manual_cleanup()) Process entered 00000020:01000000:2.0:1773941498.752488:0:14663:0:(obd_config.c:2502:class_manual_cleanup()) Manual cleanup of lustre-OST0000-osc-ffff988d08f0a000 (flags='') 00000020:00000010:2.0:1773941498.752493:0:14663:0:(obd_config.c:2506:class_manual_cleanup()) kmalloced '(lcfg)': 88 at ffff988d08bbbb40. 00000020:00000080:2.0:1773941498.752497:0:14663:0:(obd_config.c:1506:class_process_config()) processing cmd: cf004 00000020:00000001:2.0:1773941498.752505:0:14663:0:(obd_config.c:936:class_cleanup()) Process entered 00000020:00000080:2.0:1773941498.752510:0:14663:0:(obd_config.c:991:class_cleanup()) lustre-OST0000-osc-ffff988d08f0a000: forcing exports to disconnect: 0/2 00000020:00000001:2.0:1773941498.752513:0:14663:0:(genops.c:1562:class_disconnect_exports()) Process entered 00000020:00080000:2.0:1773941498.752516:0:14663:0:(genops.c:1577:class_disconnect_exports()) OBD device 5 (ffff988d06460000) has no exports 00000020:00000001:2.0:1773941498.752518:0:14663:0:(genops.c:1578:class_disconnect_exports()) Process leaving 00000020:00000001:2.0:1773941498.752522:0:14663:0:(obd_class.h:604:obd_precleanup()) Process entered 00000020:00000010:2.0:1773941498.752525:0:14663:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d0350ec00. 00000020:00000010:2.0:1773941498.752529:0:14663:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d0350f000. 00000020:00000010:2.0:1773941498.752533:0:14663:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d08bbb6c0. 00000008:00000010:2.0:1773941498.752538:0:14663:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff988d18285940. 00020000:00000010:2.0:1773941498.752542:0:14663:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff988d035a1300. 00008000:00000010:2.0:1773941498.752546:0:14663:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d08324710. 00000020:00000010:2.0:1773941498.752552:0:14663:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff988d05437e00. 00000008:00000001:2.0:1773941498.752557:0:14663:0:(osc_dev.c:163:osc_device_fini()) Process entered 00000008:00000001:2.0:1773941498.752559:0:14663:0:(osc_request.c:4123:osc_precleanup_common()) Process entered 00000008:00000001:2.0:1773941498.752566:0:14663:0:(obd_class.h:652:obd_cleanup_client_import()) Process entered 00000008:01000000:2.0:1773941498.752568:0:14663:0:(obd_class.h:663:obd_cleanup_client_import()) lustre-OST0000-osc-ffff988d08f0a000: client import never connected 00000100:00000001:2.0:1773941498.752572:0:14663:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941498.752574:0:14663:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-OST0000_UUID INVALID 00000100:00000001:2.0:1773941498.752576:0:14663:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00000001:2.0:1773941498.752586:0:14663:0:(client.c:3065:ptlrpc_free_committed()) Process entered 00000100:00100000:2.0:1773941498.752589:0:14663:0:(client.c:3081:ptlrpc_free_committed()) lustre-OST0000-osc-ffff988d08f0a000: committing for last_committed 0 gen 3 00000100:00000040:2.0:1773941498.752592:0:14663:0:(client.c:3191:ptlrpc_free_committed()) lustre-OST0000-osc-ffff988d08f0a000: normal: skip=0 replay=0/0 committed=0/0 00000100:00000001:2.0:1773941498.752594:0:14663:0:(client.c:3192:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1773941498.752597:0:14663:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941498.752599:0:14663:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941498.752601:0:14663:0:(obd_class.h:1215:obd_import_event()) Process entered 00000008:00000001:2.0:1773941498.752603:0:14663:0:(osc_request.c:3880:osc_import_event()) Process entered 00000008:00000001:2.0:1773941498.752605:0:14663:0:(osc_request.c:3947:osc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.752606:0:14663:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00100000:2.0:1773941498.752609:0:14663:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941498.752612:0:14663:0:(obd_class.h:1215:obd_import_event()) Process entered 00000008:00000001:2.0:1773941498.752614:0:14663:0:(osc_request.c:3880:osc_import_event()) Process entered 00000001:00000001:2.0:1773941498.752616:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941498.752618:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941498.752669:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:2.0:1773941498.752672:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941498.752674:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941498.752724:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941498.752727:0:14663:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:2.0:1773941498.752731:0:14663:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff988d05490240. 00000020:00000010:2.0:1773941498.752735:0:14663:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d0350fa00. 00000020:00000010:2.0:1773941498.752739:0:14663:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d0350f400. 00000020:00000010:2.0:1773941498.752743:0:14663:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d08bbb5a0. 00000008:00000010:2.0:1773941498.752749:0:14663:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff988d18286f90. 00020000:00000010:2.0:1773941498.752753:0:14663:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff988d035a1c00. 00008000:00000010:2.0:1773941498.752757:0:14663:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d08326350. 00000020:00000010:2.0:1773941498.752762:0:14663:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d0350ee00. 00000008:00000010:2.0:1773941498.752766:0:14663:0:(osc_dev.c:115:osc_session_init()) slab-alloced 'info': 352 at ffff988d03544420. 00020000:00000010:2.0:1773941498.752770:0:14663:0:(lov_dev.c:116:lov_session_key_init()) slab-alloced 'info': 576 at ffff988d02ca0000. 00008000:00000010:2.0:1773941498.752774:0:14663:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff988d111ef598. 00000020:00000001:2.0:1773941498.752777:0:14663:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446630330179322440 : -113743530229176 : ffff988d05490248) 00000020:00001000:2.0:1773941498.752779:0:14663:0:(cl_object.c:831:cl_env_get()) 1@ffff988d05490240 00000008:00000001:2.0:1773941498.752781:0:14663:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:2.0:1773941498.752783:0:14663:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:2.0:1773941498.752784:0:14663:0:(osc_cache.c:2316:osc_next_obj()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1773941498.752793:0:14663:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000001:00000001:2.0:1773941498.752795:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941498.752796:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941498.752829:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:2.0:1773941498.752833:0:14663:0:(cl_object.c:925:cl_env_put()) 1@ffff988d05490240 00000020:00000010:2.0:1773941498.752836:0:14663:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d0350f400. 00000020:00000010:2.0:1773941498.752840:0:14663:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d08bbb5a0. 00000008:00000010:2.0:1773941498.752844:0:14663:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff988d18286f90. 00020000:00000010:2.0:1773941498.752848:0:14663:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff988d035a1c00. 00008000:00000010:2.0:1773941498.752852:0:14663:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d08326350. 00000020:00000010:2.0:1773941498.752856:0:14663:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0350fa00. 00000008:00000010:2.0:1773941498.752861:0:14663:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff988d03544420. 00020000:00000010:2.0:1773941498.752865:0:14663:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff988d02ca0000. 00008000:00000010:2.0:1773941498.752869:0:14663:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff988d111ef598. 00000020:00000010:2.0:1773941498.752874:0:14663:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0350ee00. 00000020:00000010:2.0:1773941498.752878:0:14663:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff988d05490240. 00000001:00000001:2.0:1773941498.752881:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941498.752884:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941498.752918:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:2.0:1773941498.752920:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941498.752922:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941498.752955:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1773941498.752958:0:14663:0:(osc_request.c:3947:osc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.752984:0:14663:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000040:2.0:1773941498.752988:0:14663:0:(genops.c:1229:class_import_get()) import ffff988d180dc800 refcount=2 obd=lustre-OST0000-osc-ffff988d08f0a000 00000020:00000001:2.0:1773941498.752991:0:14663:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941498.752994:0:14663:0:(genops.c:1242:class_import_put()) import ffff988d180dc800 refcount=1 obd=lustre-OST0000-osc-ffff988d08f0a000 00000020:00000001:2.0:1773941498.752997:0:14663:0:(genops.c:1249:class_import_put()) Process leaving 00000020:00000001:2.0:1773941498.752999:0:14663:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941498.753001:0:14663:0:(genops.c:1242:class_import_put()) import ffff988d180dc800 refcount=0 obd=lustre-OST0000-osc-ffff988d08f0a000 00000020:00000040:2.0:1773941498.753004:0:14663:0:(genops.c:1245:class_import_put()) final put import ffff988d180dc800 00000020:00000001:2.0:1773941498.753015:0:14663:0:(genops.c:1249:class_import_put()) Process leaving 00000008:00000001:2.0:1773941498.753017:0:14663:0:(obd_class.h:670:obd_cleanup_client_import()) Process leaving 00000008:00000001:2.0:1773941498.753019:0:14663:0:(osc_request.c:4138:osc_precleanup_common()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941498.753036:0:461:0:(genops.c:1200:obd_zombie_import_free()) Process entered 00000020:00000080:0.0:1773941498.753040:0:461:0:(genops.c:1202:obd_zombie_import_free()) destroying import ffff988d180dc800 for lustre-OST0000-osc-ffff988d08f0a000 00000020:00000040:0.0:1773941498.753046:0:461:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 10 to 192.168.204.114@tcp 00000020:00000040:0.0:1773941498.753051:0:461:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 9 to 192.168.204.114@tcp 00000020:00000010:0.0:1773941498.753054:0:461:0:(genops.c:1213:obd_zombie_import_free()) kfreed 'imp_conn': 88 at ffff988d20936c00. 00000020:00000040:0.0:1773941498.753058:0:461:0:(obd_config.c:1058:class_decref()) Decref lustre-OST0000-osc-ffff988d08f0a000 (ffff988d06460000) now 4 - import 00000020:00000010:0.0:1773941498.753068:0:461:0:(genops.c:1220:obd_zombie_import_free()) kfreed 'imp': 1712 at ffff988d180dc800. 00000020:00000001:0.0:1773941498.753074:0:461:0:(genops.c:1221:obd_zombie_import_free()) Process leaving 00000020:00000010:2.0:1773941498.753319:0:14663:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 4608 at ffff988d10804000. 00000020:00000010:2.0:1773941498.753325:0:14663:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 4608 at ffff988d18bf8000. 00000020:00000010:2.0:1773941498.753337:0:14663:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 3680 at ffff988d02ece000. 00000020:00000010:2.0:1773941498.753343:0:14663:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff988d111d0800. 00000008:00000001:2.0:1773941498.753346:0:14663:0:(osc_dev.c:168:osc_device_fini()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1773941498.753351:0:14663:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff988d05437e00. 00000020:00000010:2.0:1773941498.753358:0:14663:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d0350f000. 00000020:00000010:2.0:1773941498.753361:0:14663:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d08bbb6c0. 00000008:00000010:2.0:1773941498.753364:0:14663:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff988d18285940. 00020000:00000010:2.0:1773941498.753367:0:14663:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff988d035a1300. 00008000:00000010:2.0:1773941498.753370:0:14663:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d08324710. 00000020:00000010:2.0:1773941498.753373:0:14663:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0350ec00. 00000020:00000001:2.0:1773941498.753376:0:14663:0:(obd_class.h:623:obd_precleanup()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:2.0:1773941498.753390:0:14663:0:(hash.c:1113:cfs_hash_destroy()) Process entered 00000001:00000010:2.0:1773941498.753397:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff988d065dd800. 00000001:00000010:2.0:1773941498.753400:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff988d065dd600. 00000001:00000010:2.0:1773941498.753402:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff988d065dd200. 00000001:00000010:2.0:1773941498.753404:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff988d065dc000. 00000001:00000010:2.0:1773941498.753407:0:14663:0:(hash.c:913:cfs_hash_buckets_free()) kfreed 'buckets': 32 at ffff988d09246740. 00000001:00000010:2.0:1773941498.753410:0:14663:0:(hash.c:1163:cfs_hash_destroy()) kfreed 'hs': 232 at ffff988d21fadc00. 00000001:00000001:2.0:1773941498.753415:0:14663:0:(hash.c:1165:cfs_hash_destroy()) Process leaving 00000020:00000040:2.0:1773941498.753417:0:14663:0:(obd_config.c:1058:class_decref()) Decref lustre-OST0000-osc-ffff988d08f0a000 (ffff988d06460000) now 3 - setup 00000020:00000001:2.0:1773941498.753420:0:14663:0:(obd_config.c:1021:class_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941498.753423:0:14663:0:(obd_config.c:1659:class_process_config()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000080:2.0:1773941498.753426:0:14663:0:(obd_config.c:1506:class_process_config()) processing cmd: cf002 00000020:00000001:2.0:1773941498.753430:0:14663:0:(obd_config.c:899:class_detach()) Process entered 00000020:00000040:2.0:1773941498.753432:0:14663:0:(obd_config.c:1058:class_decref()) Decref lustre-OST0000-osc-ffff988d08f0a000 (ffff988d06460000) now 2 - obd_device_list 00000020:00000080:2.0:1773941498.753435:0:14663:0:(obd_config.c:919:class_detach()) detach on obd lustre-OST0000-osc-ffff988d08f0a000 (uuid e9ee4b2c-2bac-44e1-850a-8bf1479379ac) 00000020:00000040:2.0:1773941498.753437:0:14663:0:(obd_config.c:1058:class_decref()) Decref lustre-OST0000-osc-ffff988d08f0a000 (ffff988d06460000) now 1 - newdev 00000020:00000040:2.0:1773941498.753441:0:14663:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d035c2800 with handle 0x8e86767622e288e from hash 00000020:00000040:2.0:1773941498.753445:0:14663:0:(genops.c:898:class_export_put()) PUTting export ffff988d035c2800 : new refcount 0 00000020:00000080:2.0:1773941498.753447:0:14663:0:(genops.c:904:class_export_put()) final put ffff988d035c2800/e9ee4b2c-2bac-44e1-850a-8bf1479379ac 00000020:00000001:2.0:1773941498.753450:0:14663:0:(lprocfs_status_server.c:686:lprocfs_exp_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941498.753453:0:14663:0:(genops.c:857:class_export_destroy()) Process entered 00000020:00000080:2.0:1773941498.753455:0:14663:0:(genops.c:863:class_export_destroy()) destroying export ffff988d035c2800/e9ee4b2c-2bac-44e1-850a-8bf1479379ac for lustre-OST0000-osc-ffff988d08f0a000 00000020:00000001:2.0:1773941498.753464:0:14663:0:(obd_class.h:1017:obd_destroy_export()) Process entered 00000020:00000001:2.0:1773941498.753465:0:14663:0:(obd_class.h:1021:obd_destroy_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1773941498.753468:0:14663:0:(genops.c:879:class_export_destroy()) kfreed_rcu 'exp': 1208 at ffff988d035c2800. 00000020:00000001:2.0:1773941498.753472:0:14663:0:(genops.c:880:class_export_destroy()) Process leaving 00000020:00000040:2.0:1773941498.753474:0:14663:0:(genops.c:437:class_free_dev()) Release obd device lustre-OST0000-osc-ffff988d08f0a000 obd_type name = osc 00000020:01000000:2.0:1773941498.753477:0:14663:0:(genops.c:440:class_free_dev()) finishing cleanup of obd lustre-OST0000-osc-ffff988d08f0a000 (e9ee4b2c-2bac-44e1-850a-8bf1479379ac) 00000020:00000001:2.0:1773941498.753479:0:14663:0:(obd_class.h:633:obd_cleanup()) Process entered 00000020:00000010:2.0:1773941498.753482:0:14663:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d0350ec00. 00000020:00000010:2.0:1773941498.753487:0:14663:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d0350f000. 00000020:00000010:2.0:1773941498.753490:0:14663:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d08bbb6c0. 00000008:00000010:2.0:1773941498.753494:0:14663:0:(osc_dev.c:88:osc_key_init()) slab-alloced 'info': 2856 at ffff988d18285940. 00020000:00000010:2.0:1773941498.753497:0:14663:0:(lov_dev.c:78:lov_key_init()) slab-alloced 'info': 192 at ffff988d035a1300. 00008000:00000010:2.0:1773941498.753502:0:14663:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d08324710. 00000008:00000010:2.0:1773941498.753510:0:14663:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff988d113fef90. 00000008:00000010:2.0:1773941498.753514:0:14663:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff988d18280000. 00000008:00000010:2.0:1773941498.753518:0:14663:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff988d08945940. 00000008:00000010:2.0:1773941498.753521:0:14663:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff988d08946f90. 00000008:00000010:2.0:1773941498.753524:0:14663:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff988d113fd940. 00000008:00000010:2.0:1773941498.753528:0:14663:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff988d113faca0. 00000008:00000010:2.0:1773941498.753532:0:14663:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff988d113f8000. 00000008:00000010:2.0:1773941498.753535:0:14663:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff988d113fa178. 00000008:00000010:2.0:1773941498.753538:0:14663:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff988d113fc2f0. 00000008:00000010:2.0:1773941498.753543:0:14663:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff988d035cba20. 00000008:00000010:2.0:1773941498.753546:0:14663:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff988d03545b80. 00000008:00000010:2.0:1773941498.753550:0:14663:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff988d071598c0. 00000008:00000010:2.0:1773941498.753554:0:14663:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff988d07158840. 00000008:00000010:2.0:1773941498.753556:0:14663:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff988d035cadc0. 00000008:00000010:2.0:1773941498.753559:0:14663:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff988d035cbb80. 00000008:00000010:2.0:1773941498.753562:0:14663:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff988d035ca000. 00000008:00000010:2.0:1773941498.753564:0:14663:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff988d035cab00. 00000008:00000010:2.0:1773941498.753567:0:14663:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff988d035cb340. 00000008:00000001:2.0:1773941498.753569:0:14663:0:(osc_request.c:4146:osc_cleanup_common()) Process entered 00010000:00000001:2.0:1773941498.753575:0:14663:0:(ldlm_lib.c:631:client_obd_cleanup()) Process entered 00010000:00000001:2.0:1773941498.753583:0:14663:0:(ldlm_resource.c:1304:ldlm_namespace_free_post()) Process entered 00010000:00000001:2.0:1773941498.753587:0:14663:0:(ldlm_pool.c:931:ldlm_pool_fini()) Process entered 00000020:00000010:2.0:1773941498.753617:0:14663:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 352 at ffff988d065dc600. 00000020:00000010:2.0:1773941498.753621:0:14663:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff988d111d3c00. 00010000:00000001:2.0:1773941498.753653:0:14663:0:(ldlm_pool.c:934:ldlm_pool_fini()) Process leaving 00000020:00000010:2.0:1773941498.753665:0:14663:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 96 at ffff988d20936ba0. 00000020:00000010:2.0:1773941498.753668:0:14663:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff988d111d1000. 00000001:00000001:2.0:1773941498.753725:0:14663:0:(hash.c:1113:cfs_hash_destroy()) Process entered 00000001:00000010:2.0:1773941498.753764:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 4272 at ffff988d20300000. 00000001:00000010:2.0:1773941498.753768:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 4272 at ffff988d20304000. 00000001:00000010:2.0:1773941498.753770:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 4272 at ffff988d03378000. 00000001:00000010:2.0:1773941498.753773:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 4272 at ffff988d0337c000. 00000001:00000010:2.0:1773941498.753776:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 4272 at ffff988d0337e000. 00000001:00000010:2.0:1773941498.753779:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 4272 at ffff988d0337a000. 00000001:00000010:2.0:1773941498.753781:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 4272 at ffff988d107fe000. 00000001:00000010:2.0:1773941498.753784:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 4272 at ffff988d107fa000. 00000001:00000010:2.0:1773941498.753787:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 4272 at ffff988d107f8000. 00000001:00000010:2.0:1773941498.753790:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 4272 at ffff988d107fc000. 00000001:00000010:2.0:1773941498.753792:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 4272 at ffff988d065ec000. 00000001:00000010:2.0:1773941498.753796:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 4272 at ffff988d065ee000. 00000001:00000010:2.0:1773941498.753799:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 4272 at ffff988d065ea000. 00000001:00000010:2.0:1773941498.753802:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 4272 at ffff988d065e8000. 00000001:00000010:2.0:1773941498.753806:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 4272 at ffff988d0646a000. 00000001:00000010:2.0:1773941498.753810:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 4272 at ffff988d0646e000. 00000001:00000010:2.0:1773941498.753812:0:14663:0:(hash.c:913:cfs_hash_buckets_free()) kfreed 'buckets': 128 at ffff988d03580380. 00000001:00000010:2.0:1773941498.753815:0:14663:0:(hash.c:1163:cfs_hash_destroy()) kfreed 'hs': 280 at ffff988d065dcc00. 00000001:00000001:2.0:1773941498.753818:0:14663:0:(hash.c:1165:cfs_hash_destroy()) Process leaving 00010000:00000010:2.0:1773941498.753821:0:14663:0:(ldlm_resource.c:1327:ldlm_namespace_free_post()) kfreed 'ns': 848 at ffff988d111d1800. 00010000:00000001:2.0:1773941498.753825:0:14663:0:(ldlm_lockd.c:3044:ldlm_put_ref()) Process entered 00010000:00000001:2.0:1773941498.753827:0:14663:0:(ldlm_lockd.c:3058:ldlm_put_ref()) Process leaving 00010000:00000001:2.0:1773941498.753829:0:14663:0:(ldlm_resource.c:1329:ldlm_namespace_free_post()) Process leaving 00010000:00000001:2.0:1773941498.753831:0:14663:0:(obd_class.h:652:obd_cleanup_client_import()) Process entered 00010000:00000001:2.0:1773941498.753833:0:14663:0:(obd_class.h:670:obd_cleanup_client_import()) Process leaving 00010000:00000001:2.0:1773941498.753835:0:14663:0:(ldlm_lockd.c:3044:ldlm_put_ref()) Process entered 00010000:00000001:2.0:1773941498.753837:0:14663:0:(ldlm_lockd.c:3058:ldlm_put_ref()) Process leaving 00010000:00000010:2.0:1773941498.753845:0:14663:0:(ldlm_lib.c:647:client_obd_cleanup()) kfreed 'cli->cl_read_io_latency_by_size': 3920 at ffff988d02ecb000. 00010000:00000010:2.0:1773941498.753850:0:14663:0:(ldlm_lib.c:651:client_obd_cleanup()) kfreed 'cli->cl_write_io_latency_by_size': 3920 at ffff988d02eca000. 00010000:00000001:2.0:1773941498.753853:0:14663:0:(ldlm_lib.c:654:client_obd_cleanup()) Process leaving 00000008:00000001:2.0:1773941498.753855:0:14663:0:(osc_request.c:4167:osc_cleanup_common()) Process leaving (rc=0 : 0 : 0) 00000008:00000010:2.0:1773941498.753858:0:14663:0:(osc_dev.c:179:osc_device_free()) kfreed 'oc': 104 at ffff988d03580b80. 00000020:00000010:2.0:1773941498.753861:0:14663:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d0350f000. 00000020:00000010:2.0:1773941498.753864:0:14663:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d08bbb6c0. 00000008:00000010:2.0:1773941498.753868:0:14663:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff988d18285940. 00020000:00000010:2.0:1773941498.753871:0:14663:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff988d035a1300. 00008000:00000010:2.0:1773941498.753874:0:14663:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d08324710. 00000020:00000010:2.0:1773941498.753878:0:14663:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0350ec00. 00000020:00000001:2.0:1773941498.753880:0:14663:0:(obd_class.h:647:obd_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1773941498.753883:0:14663:0:(genops.c:66:obd_device_free()) slab-freed '(obd)': 6584 at ffff988d06460000. 00000020:00000001:2.0:1773941498.753888:0:14663:0:(obd_config.c:923:class_detach()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941498.753891:0:14663:0:(obd_config.c:1655:class_process_config()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1773941498.753894:0:14663:0:(obd_config.c:2523:class_manual_cleanup()) kfreed 'lcfg': 88 at ffff988d08bbbb40. 00000020:00000001:2.0:1773941498.753897:0:14663:0:(obd_config.c:2524:class_manual_cleanup()) Process leaving (rc=0 : 0 : 0) 00020000:00000010:2.0:1773941498.753901:0:14663:0:(lu_tgt_descs.c:285:lu_tgt_descs_fini()) kfreed 'ltd->ltd_tgt_idx[i]': 4096 at ffff988d02ec8000. 00020000:00000001:2.0:1773941498.753907:0:14663:0:(lov_obd.c:795:lov_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1773941498.753917:0:14663:0:(lu_object.c:1212:lu_site_fini()) kvfree 's->ls_bkts': 18432 at ffff988d05408000. 00000020:00000010:2.0:1773941498.753929:0:14663:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 192 at ffff988d02eb6600. 00000020:00000010:2.0:1773941498.753933:0:14663:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff988d065b0c00. 00020000:00000010:2.0:1773941498.753938:0:14663:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff988d20218180. 00020000:00000010:2.0:1773941498.753941:0:14663:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff988d035a1000. 00020000:00000010:2.0:1773941498.753944:0:14663:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff988d180f76c0. 00020000:00000010:2.0:1773941498.753947:0:14663:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff988d180f7e40. 00020000:00000010:2.0:1773941498.753950:0:14663:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff988d20218840. 00020000:00000010:2.0:1773941498.753953:0:14663:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff988d20218780. 00020000:00000010:2.0:1773941498.753955:0:14663:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff988d20218c00. 00020000:00000010:2.0:1773941498.753958:0:14663:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff988d20218540. 00010000:00000010:2.1:1773941498.753984:0:14663:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d071eec00. 00010000:00000010:2.1:1773941498.753995:0:14663:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d089dfc00. 00010000:00000010:1.1F:1773941498.753996:0:14670:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d0168a600. 00020000:00000010:2.0:1773941498.754010:0:14663:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff988d20218a80. 00010000:00000010:1.1:1773941498.754014:0:14670:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d1a01da00. 00020000:00000010:2.0:1773941498.754017:0:14663:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff988d113f6400. 00010000:00000010:1.1:1773941498.754023:0:14670:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d0168b400. 00020000:00000010:2.0:1773941498.754025:0:14663:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff988d02ca1f80. 00020000:00000010:2.0:1773941498.754032:0:14663:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff988d182aaac0. 00020000:00000010:2.0:1773941498.754043:0:14663:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff988d182a8240. 00020000:00000010:2.0:1773941498.754048:0:14663:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff988d113f5d40. 00020000:00000010:2.0:1773941498.754053:0:14663:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff988d113f4d80. 00020000:00000010:2.0:1773941498.754058:0:14663:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff988d113f4b40. 00020000:00000010:2.0:1773941498.754062:0:14663:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff988d113f6f40. 00020000:00000010:2.0:1773941498.754067:0:14663:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff988d113f6640. 00020000:00000010:2.0:1773941498.754072:0:14663:0:(lov_dev.c:306:lov_device_free()) kfreed 'ld->ld_target': 512 at ffff988d065dc200. 00020000:00000010:2.0:1773941498.754078:0:14663:0:(lov_dev.c:310:lov_device_free()) kfreed 'ld->ld_md_tgts': 4096 at ffff988d02d6e000. 00020000:00000010:2.0:1773941498.754082:0:14663:0:(lov_dev.c:315:lov_device_free()) kfreed 'ld->ld_lov->lov_mdc_tgts': 4096 at ffff988d02d68000. 00020000:00000010:2.0:1773941498.754087:0:14663:0:(lov_dev.c:319:lov_device_free()) kfreed 'ld': 632 at ffff988d054e1000. 00000020:00001000:2.0:1773941498.754093:0:14663:0:(cl_object.c:925:cl_env_put()) 1@ffff988d08a3f3f0 00000020:00000010:2.0:1773941498.754096:0:14663:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d1817ce00. 00000020:00000010:2.0:1773941498.754100:0:14663:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d111f7540. 00000008:00000010:2.0:1773941498.754105:0:14663:0:(osc_dev.c:100:osc_key_fini()) slab-freed '(info)': 2856 at ffff988d113f9650. 00020000:00000010:2.0:1773941498.754111:0:14663:0:(lov_dev.c:94:lov_key_fini()) slab-freed '(info)': 192 at ffff988d20218d80. 00008000:00000010:2.0:1773941498.754116:0:14663:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d107f2de8. 00000080:00000010:2.0:1773941498.754123:0:14663:0:(vvp_dev.c:80:ll_thread_key_fini()) slab-freed '(lti)': 168 at ffff988d181e7930. 00000080:00000010:2.0:1773941498.754129:0:14663:0:(vvp_dev.c:130:vvp_thread_key_fini()) slab-freed '(vti)': 584 at ffff988d054abb50. 00000020:00000010:2.0:1773941498.754135:0:14663:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d1817cc00. 00000008:00000010:2.0:1773941498.754139:0:14663:0:(osc_dev.c:125:osc_session_fini()) slab-freed '(info)': 352 at ffff988d035ca9a0. 00020000:00000010:2.0:1773941498.754144:0:14663:0:(lov_dev.c:135:lov_session_key_fini()) slab-freed '(info)': 576 at ffff988d113f6880. 00008000:00000010:2.0:1773941498.754148:0:14663:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff988d209199e8. 00000080:00000010:2.0:1773941498.754153:0:14663:0:(vvp_dev.c:105:vvp_session_key_fini()) slab-freed '(session)': 240 at ffff988d071534b0. 00000020:00000010:2.0:1773941498.754158:0:14663:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d1817c000. 00000020:00000010:2.0:1773941498.754162:0:14663:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff988d08a3f3f0. 00000080:00000001:2.0:1773941498.754166:0:14663:0:(vvp_dev.c:331:cl_sb_fini()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1773941498.754169:0:14663:0:(obd_class.h:902:obd_disconnect()) Process entered 00020000:00000001:2.0:1773941498.754172:0:14663:0:(lov_obd.c:290:lov_disconnect()) Process entered 00000020:00000001:2.0:1773941498.754174:0:14663:0:(genops.c:1462:class_disconnect()) Process entered 00000020:00000080:2.0:1773941498.754176:0:14663:0:(genops.c:1487:class_disconnect()) disconnect: cookie 0x8e86767622e28b8 00000020:00000040:2.0:1773941498.754180:0:14663:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d035c5000 with handle 0x8e86767622e28b8 from hash 00000020:00000001:2.0:1773941498.754185:0:14663:0:(genops.c:1934:obd_stale_export_put()) Process entered 00000020:00000040:2.0:1773941498.754187:0:14663:0:(genops.c:898:class_export_put()) PUTting export ffff988d035c5000 : new refcount 1 00000020:00000001:2.0:1773941498.754190:0:14663:0:(genops.c:1959:obd_stale_export_put()) Process leaving 00000020:00000040:2.0:1773941498.754197:0:14663:0:(genops.c:898:class_export_put()) PUTting export ffff988d035c5000 : new refcount 0 00000020:00000080:2.0:1773941498.754199:0:14663:0:(genops.c:904:class_export_put()) final put ffff988d035c5000/e9ee4b2c-2bac-44e1-850a-8bf1479379ac 00000020:00000001:2.0:1773941498.754202:0:14663:0:(lprocfs_status_server.c:686:lprocfs_exp_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941498.754228:0:14663:0:(genops.c:1493:class_disconnect()) Process leaving (rc=0 : 0 : 0) 00020000:00000001:2.0:1773941498.754231:0:14663:0:(lov_obd.c:321:lov_disconnect()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:2.0:1773941498.754234:0:14663:0:(obd_class.h:914:obd_disconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941498.754296:0:8:0:(genops.c:857:class_export_destroy()) Process entered 00000020:00000080:0.0:1773941498.754299:0:8:0:(genops.c:863:class_export_destroy()) destroying export ffff988d035c5000/e9ee4b2c-2bac-44e1-850a-8bf1479379ac for lustre-clilov-ffff988d08f0a000 00000020:00000001:0.0:1773941498.754301:0:8:0:(obd_class.h:1017:obd_destroy_export()) Process entered 00000020:00000001:0.0:1773941498.754303:0:8:0:(obd_class.h:1021:obd_destroy_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1773941498.754305:0:8:0:(obd_config.c:1058:class_decref()) Decref lustre-clilov-ffff988d08f0a000 (ffff988d089a99b8) now 4 - export 00000020:00000010:0.0:1773941498.754307:0:8:0:(genops.c:879:class_export_destroy()) kfreed_rcu 'exp': 1208 at ffff988d035c5000. 00000020:00000001:0.0:1773941498.754313:0:8:0:(genops.c:880:class_export_destroy()) Process leaving 00000020:00000010:2.0:1773941498.754551:0:14663:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 576 at ffff988d035a6c00. 00000020:00000010:2.0:1773941498.754556:0:14663:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff988d035a6000. 00000020:00000010:2.0:1773941498.754559:0:14663:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 1856 at ffff988d111b5800. 00000020:00000010:2.0:1773941498.754562:0:14663:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 1856 at ffff988d18b19800. 00000020:00000010:2.0:1773941498.754565:0:14663:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 1440 at ffff988d08f0e800. 00000020:00000010:2.0:1773941498.754569:0:14663:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff988d035a5400. 00000080:00000001:2.0:1773941498.754572:0:14663:0:(obd_class.h:902:obd_disconnect()) Process entered 00800000:00000001:2.0:1773941498.754575:0:14663:0:(lmv_obd.c:589:lmv_disconnect()) Process entered 00800000:00000001:2.0:1773941498.754578:0:14663:0:(lmv_obd.c:534:lmv_disconnect_mdc()) Process entered 00020000:00000001:2.0:1773941498.754585:0:14663:0:(lu_tgt_descs.c:164:lu_qos_del_tgt()) Process entered 00020000:00000001:2.0:1773941498.754588:0:14663:0:(lu_tgt_descs.c:186:lu_qos_del_tgt()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:2.0:1773941498.754591:0:14663:0:(fld_request.c:185:fld_client_del_target()) Process entered 00000020:00000040:2.0:1773941498.754594:0:14663:0:(genops.c:898:class_export_put()) PUTting export ffff988d035c0800 : new refcount 3 80000000:00000010:2.0:1773941498.754597:0:14663:0:(fld_request.c:197:fld_client_del_target()) kfreed 'target': 40 at ffff988d082b20c0. 80000000:00000001:2.0:1773941498.754600:0:14663:0:(fld_request.c:198:fld_client_del_target()) Process leaving (rc=0 : 0 : 0) 40000000:00000001:2.0:1773941498.754603:0:14663:0:(fid_request.c:460:client_fid_fini()) Process entered 40000000:00000001:2.0:1773941498.754605:0:14663:0:(fid_request.c:376:seq_client_fini()) Process entered 00000020:00000040:2.0:1773941498.754644:0:14663:0:(genops.c:898:class_export_put()) PUTting export ffff988d035c0800 : new refcount 2 40000000:00000001:2.0:1773941498.754647:0:14663:0:(fid_request.c:386:seq_client_fini()) Process leaving 40000000:00000010:2.0:1773941498.754649:0:14663:0:(fid_request.c:465:client_fid_fini()) kfreed 'cli->cl_seq': 224 at ffff988d21fad000. 40000000:00000001:2.0:1773941498.754653:0:14663:0:(fid_request.c:470:client_fid_fini()) Process leaving (rc=0 : 0 : 0) 00800000:00000040:2.0:1773941498.754655:0:14663:0:(lmv_obd.c:568:lmv_disconnect_mdc()) Disconnected from lustre-MDT0000-mdc-ffff988d08f0a000(e9ee4b2c-2bac-44e1-850a-8bf1479379ac) successfully 00800000:00000001:2.0:1773941498.754658:0:14663:0:(obd_class.h:1347:obd_register_observer()) Process entered 00800000:00000001:2.0:1773941498.754661:0:14663:0:(obd_class.h:1360:obd_register_observer()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:2.0:1773941498.754663:0:14663:0:(obd_class.h:902:obd_disconnect()) Process entered 00010000:00000001:2.0:1773941498.754666:0:14663:0:(ldlm_lib.c:759:client_disconnect_export()) Process entered 00010000:00000040:2.0:1773941498.754669:0:14663:0:(ldlm_lib.c:772:client_disconnect_export()) disconnect lustre-MDT0000-mdc-ffff988d08f0a000 - 1 00000100:00000001:2.0:1773941498.754671:0:14663:0:(pinger.c:388:ptlrpc_pinger_del_import()) Process entered 00000100:00080000:2.0:1773941498.754674:0:14663:0:(pinger.c:396:ptlrpc_pinger_del_import()) removing pingable import e9ee4b2c-2bac-44e1-850a-8bf1479379ac->lustre-MDT0000_UUID 00000020:00000001:2.0:1773941498.754682:0:14663:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941498.754684:0:14663:0:(genops.c:1242:class_import_put()) import ffff988d035c7000 refcount=2 obd=lustre-MDT0000-mdc-ffff988d08f0a000 00000020:00000001:2.0:1773941498.754687:0:14663:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941498.754688:0:14663:0:(pinger.c:401:ptlrpc_pinger_del_import()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941498.754691:0:14663:0:(ldlm_request.c:2367:ldlm_cli_cancel_unused()) Process entered 00000001:00000001:2.0:1773941498.754693:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941498.754695:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00010000:00000040:2.0:1773941498.754822:0:14663:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff988d21ec1800 count: 2 00010000:00000001:2.0:1773941498.754826:0:14663:0:(ldlm_request.c:2311:ldlm_cli_cancel_unused_resource()) Process entered 00010000:00000040:2.0:1773941498.754830:0:14663:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff988d21ec1800 count: 3 00010000:00000001:2.0:1773941498.754833:0:14663:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:2.0:1773941498.754836:0:14663:0:(ldlm_request.c:2161:ldlm_cancel_resource_local()) Process entered 00010000:00000001:2.0:1773941498.754840:0:14663:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:2.0:1773941498.754845:0:14663:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff988d08f0a000 lock: ffff988d00030000/0x8e86767622e28c6 lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0xd03f0241a6d949fd expref: -99 pid: 14647 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00000080:00000001:2.0:1773941498.754856:0:14663:0:(namei.c:472:ll_md_blocking_ast()) Process entered 00000080:00000001:2.0:1773941498.754859:0:14663:0:(namei.c:239:ll_lock_cancel_bits()) Process entered 00000080:00000001:2.0:1773941498.754861:0:14663:0:(namei.c:251:ll_lock_cancel_bits()) Process leaving 00000080:00000001:2.0:1773941498.754863:0:14663:0:(namei.c:536:ll_md_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941498.754867:0:14663:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:2.0:1773941498.754870:0:14663:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1773941498.754872:0:14663:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1773941498.754874:0:14663:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941498.754878:0:14663:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1773941498.754881:0:14663:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d00030000 with handle 0x8e86767622e28c6 from hash 00010000:00000001:2.0:1773941498.754884:0:14663:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1773941498.754885:0:14663:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941498.754887:0:14663:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941498.754888:0:14663:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000020:00000010:2.0:1773941498.754893:0:14663:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 448 at ffff988d0350ec00. 00010000:00000001:2.0:1773941498.754896:0:14663:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:2.0:1773941498.754898:0:14663:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:2.0:1773941498.754901:0:14663:0:(ldlm_request.c:1740:ldlm_cli_cancel_list_local()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1773941498.754903:0:14663:0:(ldlm_request.c:2211:ldlm_cancel_resource_local()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1773941498.754911:0:14663:0:(ldlm_request.c:2242:ldlm_cli_cancel_list()) Process entered 00010000:00000001:2.0:1773941498.754914:0:14663:0:(ldlm_request.c:1396:ldlm_cli_cancel_req()) Process entered 00000100:00000010:2.0:1773941498.754920:0:14663:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06505c00. 00000020:00000040:2.0:1773941498.754923:0:14663:0:(genops.c:1229:class_import_get()) import ffff988d035c7000 refcount=3 obd=lustre-MDT0000-mdc-ffff988d08f0a000 00000100:00000001:2.0:1773941498.754928:0:14663:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941498.754931:0:14663:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941498.754934:0:14663:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941498.754939:0:14663:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d0350f000. 00000100:00000001:2.0:1773941498.754944:0:14663:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941498.754947:0:14663:0:(ldlm_request.c:1348:ldlm_cancel_pack()) Process entered 00010000:00010000:2.0:1773941498.754954:0:14663:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff988d08f0a000 lock: ffff988d00030000/0x8e86767622e28c6 lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xd03f0241a6d949fd expref: -99 pid: 14647 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00010000:00010000:2.0:1773941498.754985:0:14663:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00000001:2.0:1773941498.754987:0:14663:0:(ldlm_request.c:1380:ldlm_cancel_pack()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941498.754991:0:14663:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:2.0:1773941498.754993:0:14663:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:2.0:1773941498.754999:0:14663:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d18807900. 00000100:00000001:2.0:1773941498.755001:0:14663:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330501724416 : -113743207827200 : ffff988d18807900) 00000100:00000001:2.0:1773941498.755004:0:14663:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941498.755006:0:14663:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196581376 : -113743512970240 : ffff988d06505c00) 00000100:00000001:2.0:1773941498.755010:0:14663:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941498.755013:0:14663:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.755015:0:14663:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:2.0:1773941498.755017:0:14663:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941498.755023:0:14663:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06505c00 x1860112291476096/t0(0) o103->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941498.755032:0:14663:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941498.755034:0:14663:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941498.755039:0:14663:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06505c00 x1860112291476096/t0(0) o103->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941498.755046:0:14663:0:(client.c:1841:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:2.0:1773941498.755049:0:14663:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941498.755057:0:14663:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941498.755061:0:14663:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d18807900 going to sleep for 0 seconds 00000100:00000001:2.0:1773941498.755063:0:14663:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941498.755066:0:14663:0:(client.c:2089:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:2.0:1773941498.755069:0:14663:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941498.755071:0:14663:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941498.755073:0:14663:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941498.755078:0:14663:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06505c00 x1860112291476096/t0(0) o103->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941498.755088:0:14663:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06505c00 pname:cluuid:pid:xid:nid:opc:job umount:e9ee4b2c-2bac-44e1-850a-8bf1479379ac:14663:1860112291476096:192.168.204.114@tcp:103:umount.0 00000100:00000001:2.0:1773941498.755093:0:14663:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941498.755096:0:14663:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:2.0:1773941498.755098:0:14663:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:2.0:1773941498.755100:0:14663:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941498.755104:0:14663:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06505c00 x1860112291476096/t0(0) o103->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941498.755112:0:14663:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941498.755114:0:14663:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d18807900. 00000100:00000001:2.0:1773941498.755117:0:14663:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:2.0:1773941498.755119:0:14663:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00010000:2.0:1773941498.755121:0:14663:0:(ldlm_request.c:1494:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00000100:00000001:2.0:1773941498.755123:0:14663:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941498.755126:0:14663:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06505c00 x1860112291476096/t0(0) o103->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941498.755133:0:14663:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000020:00000001:2.0:1773941498.755135:0:14663:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941498.755137:0:14663:0:(genops.c:1242:class_import_put()) import ffff988d035c7000 refcount=2 obd=lustre-MDT0000-mdc-ffff988d08f0a000 00000020:00000001:2.0:1773941498.755139:0:14663:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941498.755142:0:14663:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d0350f000. 02000000:00000001:2.0:1773941498.755145:0:14663:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941498.755146:0:14663:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941498.755148:0:14663:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06505c00. 00000100:00000001:2.0:1773941498.755156:0:14663:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941498.755157:0:14663:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1773941498.755159:0:14663:0:(ldlm_request.c:1501:ldlm_cli_cancel_req()) Process leaving 00010000:00010000:2.0:1773941498.755161:0:14663:0:(ldlm_request.c:2277:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00000001:2.0:1773941498.755163:0:14663:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1773941498.755167:0:14663:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff988d08f0a000 lock: ffff988d00030000/0x8e86767622e28c6 lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0xd03f0241a6d949fd expref: -99 pid: 14647 timeout: 0 lvb_type: 0 lru_score: 5 lru_type: 1 00000020:00000010:2.0:1773941498.755177:0:14663:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 128 at ffff988d107f4f80. 00010000:00000001:2.0:1773941498.755182:0:14663:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941498.755184:0:14663:0:(ldlm_request.c:2290:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941498.755186:0:14663:0:(ldlm_request.c:2328:ldlm_cli_cancel_unused_resource()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1773941498.755190:0:14663:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d21ec1800 count: 0 00010000:00000010:2.0:1773941498.755195:0:14663:0:(ldlm_resource.c:1529:ldlm_resource_free()) kfreed 'res->lr_ibits_queues': 112 at ffff988d181feb00. 00000001:00000001:2.0:1773941498.755587:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941498.755590:0:14663:0:(ldlm_request.c:2379:ldlm_cli_cancel_unused()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941498.755593:0:14663:0:(ldlm_resource.c:1268:ldlm_namespace_free_prior()) Process entered 00010000:00000001:2.0:1773941498.755595:0:14663:0:(ldlm_resource.c:1207:__ldlm_namespace_free()) Process entered 00000001:00000001:2.0:1773941498.755597:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941498.755599:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941498.756066:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:2.0:1773941498.756069:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941498.756070:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941498.756523:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941498.756526:0:14663:0:(ldlm_resource.c:1251:__ldlm_namespace_free()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941498.756529:0:14663:0:(ldlm_resource.c:1293:ldlm_namespace_free_prior()) Process leaving 00000100:00000001:2.0:1773941498.756531:0:14663:0:(import.c:1924:ptlrpc_disconnect_import()) Process entered 00000100:00000001:2.0:1773941498.756535:0:14663:0:(import.c:1858:ptlrpc_disconnect_import_async()) Process entered 00000100:00000001:2.0:1773941498.756537:0:14663:0:(import.c:1750:ptlrpc_disconnect_prep_req()) Process entered 00000100:00000010:2.0:1773941498.756541:0:14663:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06505c00. 00000020:00000040:2.0:1773941498.756545:0:14663:0:(genops.c:1229:class_import_get()) import ffff988d035c7000 refcount=3 obd=lustre-MDT0000-mdc-ffff988d08f0a000 00000100:00000001:2.0:1773941498.756548:0:14663:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941498.756551:0:14663:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941498.756553:0:14663:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941498.756558:0:14663:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff988d217a8e00. 00000100:00000001:2.0:1773941498.756561:0:14663:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.756570:0:14663:0:(import.c:1788:ptlrpc_disconnect_prep_req()) Process leaving (rc=18446630330196581376 : -113743512970240 : ffff988d06505c00) 00000100:00080000:2.0:1773941498.756573:0:14663:0:(import.c:65:import_set_state_nolock()) ffff988d035c7000 lustre-MDT0000_UUID: changing import state from FULL to CONNECTING 00000100:00000001:2.0:1773941498.756578:0:14663:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941498.756580:0:14663:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941498.756588:0:14663:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005fe92633] to pc [ptlrpcd_rcv+-1] req@ffff988d06505c00 x1860112291476224/t0(0) o39->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941498.756609:0:14663:0:(import.c:1904:ptlrpc_disconnect_import_async()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.756629:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941498.756633:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941498.756634:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941498.756639:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06505c00 x1860112291476224/t0(0) o39->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941498.756645:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941498.756646:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941498.756648:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941498.756650:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941498.756652:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941498.756654:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06505c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9ee4b2c-2bac-44e1-850a-8bf1479379ac:2400:1860112291476224:192.168.204.114@tcp:39:umount.0 00000100:00000001:0.0:1773941498.756657:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941498.756661:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941498.756662:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941498.756664:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941498.756668:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff988d0381f200. 02000000:00000001:0.0:1773941498.756670:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941498.756675:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941498.756679:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f420. 00000400:00000010:0.0:1773941498.756684:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833aa0. 00000100:00000200:0.0:1773941498.756689:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112291476224, portal 10 00000100:00000001:0.0:1773941498.756690:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941498.756691:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196581376 : -113743512970240 : ffff988d06505c00) 00000100:00000040:0.0:1773941498.756696:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06505c00 x1860112291476224/t0(0) o39->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941508 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941498.756706:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941498.756708:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941498.756710:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833908. 00000100:00000200:0.0:1773941498.756713:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1860112291476224, offset 0 00000400:00000200:0.0:1773941498.756718:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941498.756724:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941498.756731:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941498.756735:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941498.756738:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941498.756741:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941498.756743:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:0.0:1773941498.756747:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [108:108:256:4294967295] 192.168.204.114@tcp LPNI seq info [108:108:8:4294967295] 00000400:00000200:0.0:1773941498.756750:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941498.756760:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:0.0:1773941498.756764:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:0.0:1773941498.756767:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d04c1a400. 00000800:00000200:0.0:1773941498.756773:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:0.0:1773941498.756779:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:0.0:1773941498.756782:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d04c1a400 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1773941498.756791:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941498.756794:0:2400:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0000-mdc-ffff988d08f0a000: using a timeout of 5210 usecs (6 jiffies) 00000400:00000200:0.0:1773941498.756797:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941498.756800:0:2400:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff988d08f0a000: PM QoS new deadline estimation for cpu 0 is 4294889103 00000100:00000001:0.0:1773941498.756802:0:2400:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff988d08f0a000: PM QoS request active for cpu 0, keep current deadline 4294890340 00000100:00000001:0.0:1773941498.756804:0:2400:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0000-mdc-ffff988d08f0a000: PM QoS new deadline estimation for cpu 1 is 4294889103 00000100:00000001:0.0:1773941498.756806:0:2400:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0000-mdc-ffff988d08f0a000: PM QoS request active for cpu 1, keep current deadline 4294890340 00000100:00000001:0.0:1773941498.756807:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941498.756809:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.756815:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.756817:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941498.756821:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941498.756822:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1773941498.756824:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941498.756825:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941498.756826:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.756827:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941498.756910:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941498.756913:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d04c1a400. 00000400:00000200:0.0:1773941498.756917:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941498.756921:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941498.756924:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833908 00000400:00000010:0.0:1773941498.756926:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833908. 00000100:00000001:0.0:1773941498.756929:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941498.756932:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06505c00 x1860112291476224/t0(0) o39->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941508 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941498.756939:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941498.756942:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06505c00 x1860112291476224/t0(0) o39->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941508 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941498.756947:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.756948:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941498.758899:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941498.758910:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941498.758913:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941498.758915:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941498.758920:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941498.758924:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941498.758928:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941498.758931:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941498.758935:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941498.758938:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 10 MB=0x69bc324603300 00000400:00000200:0.0:1773941498.758942:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941498.758945:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.114@tcp of length 224/224 into md 0x8a1 [1] + 224 00000800:00000001:0.0:1773941498.758957:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941498.758988:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941498.758991:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941498.758994:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941498.758997:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941498.759002:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06505c00 x1860112291476224/t0(0) o39->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941508 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941498.759010:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff988d06505c00 x1860112291476224/t0(0) o39->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941508 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941498.759027:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941498.759034:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941498.759037:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941498.759049:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941498.759052:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941498.759057:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941498.759059:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941498.759062:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f420. 00000400:00000200:2.0:1773941498.759066:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833aa0 00000400:00000010:2.0:1773941498.759068:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833aa0. 00000100:00000001:2.0:1773941498.759072:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941498.759078:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06505c00 x1860112291476224/t0(0) o39->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941508 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941498.759091:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06505c00 x1860112291476224/t0(0) o39->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941508 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941498.759099:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941498.759101:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941498.759104:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941498.759106:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941498.759109:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941498.759111:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941498.759114:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941498.759120:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d035c7458 time=1 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941498.759124:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d035c7410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941498.759128:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941498.759129:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941498.759132:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941498.759135:0:2400:0:(ldlm_request.c:1546:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.759139:0:2400:0:(client.c:1532:ptlrpc_save_versions()) Process entered 00000100:00000040:2.0:1773941498.759141:0:2400:0:(client.c:1539:ptlrpc_save_versions()) Client save versions [0x0/0x0] 00000100:00000001:2.0:1773941498.759149:0:2400:0:(client.c:1541:ptlrpc_save_versions()) Process leaving 00000100:00000001:2.0:1773941498.759151:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941498.759153:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196581376 : -113743512970240 : ffff988d06505c00) 00000100:00000001:2.0:1773941498.759156:0:2400:0:(client.c:3065:ptlrpc_free_committed()) Process entered 00000100:00000040:2.0:1773941498.759158:0:2400:0:(client.c:3077:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff988d08f0a000: skip recheck: last_committed 0 00000100:00000001:2.0:1773941498.759160:0:2400:0:(client.c:3078:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1773941498.759162:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941498.759167:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06505c00 x1860112291476224/t8589934593(8589934593) o39->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941508 ref 2 fl Rpc:RNQU/204/0 rc 0/0 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941498.759177:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941498.759179:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941498.759181:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1773941498.759184:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d035c7000 lustre-MDT0000_UUID: changing import state from CONNECTING to CLOSED 00000100:00000001:2.0:1773941498.759187:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 00000002:00000001:2.0:1773941498.759189:0:2400:0:(mdc_request.c:2862:mdc_import_event()) Process entered 00000002:00000001:2.0:1773941498.759192:0:2400:0:(mdc_request.c:2936:mdc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.759194:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941498.759195:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 00000002:00000001:2.0:1773941498.759197:0:2400:0:(mdc_request.c:2862:mdc_import_event()) Process entered 00000002:00000001:2.0:1773941498.759199:0:2400:0:(mdc_request.c:2936:mdc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.759201:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000040:2.0:1773941498.759226:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06505c00 x1860112291476224/t8589934593(8589934593) o39->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941508 ref 2 fl Interpret:RNQU/204/0 rc 0/0 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941498.759237:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06505c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9ee4b2c-2bac-44e1-850a-8bf1479379ac:2400:1860112291476224:192.168.204.114@tcp:39:umount.0 00000100:00000001:3.0:1773941498.759243:0:14663:0:(import.c:1967:ptlrpc_disconnect_import()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.759244:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941498.759250:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1773941498.759251:0:14663:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941498.759253:0:14663:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-MDT0000_UUID INVALID 00000100:00000001:3.0:1773941498.759256:0:14663:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00000040:2.0:1773941498.759256:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06505c00 x1860112291476224/t8589934593(8589934593) o39->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941508 ref 2 fl Complete:RNQU/204/0 rc 0/0 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941498.759259:0:14663:0:(client.c:3065:ptlrpc_free_committed()) Process entered 00000100:00100000:3.0:1773941498.759260:0:14663:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff988d08f0a000: committing for last_committed 0 gen 2 00000100:00100000:3.0:1773941498.759269:0:14663:0:(client.c:3119:ptlrpc_free_committed()) @@@ free request with old gen req@ffff988d06505c00 x1860112291476224/t8589934593(8589934593) o39->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941508 ref 2 fl Complete:RNQU/204/0 rc 0/0 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941498.759272:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.759279:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941498.759281:0:14663:0:(client.c:3120:ptlrpc_free_committed()) Process leaving via free_req (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941498.759285:0:14663:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941498.759286:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:3.0:1773941498.759288:0:14663:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06505c00 x1860112291476224/t8589934593(8589934593) o39->lustre-MDT0000-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941508 ref 1 fl Complete:RNQU/204/0 rc 0/0 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941498.759289:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941498.759295:0:14663:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941498.759297:0:14663:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941498.759301:0:14663:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff988d0381f200. 00000100:00000001:2.0:1773941498.759302:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1773941498.759304:0:14663:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1773941498.759306:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941498.759307:0:14663:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941498.759310:0:14663:0:(genops.c:1242:class_import_put()) import ffff988d035c7000 refcount=2 obd=lustre-MDT0000-mdc-ffff988d08f0a000 00000020:00000001:3.0:1773941498.759312:0:14663:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941498.759315:0:14663:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff988d217a8e00. 02000000:00000001:3.0:1773941498.759317:0:14663:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941498.759319:0:14663:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941498.759320:0:14663:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06505c00. 00000100:00000001:3.0:1773941498.759322:0:14663:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941498.759323:0:14663:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941498.759325:0:14663:0:(client.c:3191:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff988d08f0a000: normal: skip=0 replay=1/1 committed=0/0 00000100:00000001:3.0:1773941498.759327:0:14663:0:(client.c:3192:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1773941498.759327:0:14663:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941498.759328:0:14663:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941498.759329:0:14663:0:(obd_class.h:1215:obd_import_event()) Process entered 00000002:00000001:3.0:1773941498.759330:0:14663:0:(mdc_request.c:2862:mdc_import_event()) Process entered 00000002:00000001:3.0:1773941498.759332:0:14663:0:(mdc_request.c:2936:mdc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941498.759333:0:14663:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00100000:3.0:1773941498.759341:0:14663:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:3.0:1773941498.759343:0:14663:0:(obd_class.h:1215:obd_import_event()) Process entered 00000002:00000001:3.0:1773941498.759344:0:14663:0:(mdc_request.c:2862:mdc_import_event()) Process entered 00000001:00000001:3.0:1773941498.759345:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941498.759347:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941498.759692:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941498.759694:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941498.759695:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941498.760045:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941498.760048:0:14663:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:3.0:1773941498.760052:0:14663:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff988d08a3fcf0. 00000020:00000010:3.0:1773941498.760056:0:14663:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d1817c600. 00000020:00000010:3.0:1773941498.760059:0:14663:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d1817d800. 00000020:00000010:3.0:1773941498.760062:0:14663:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d111f7780. 00008000:00000010:3.0:1773941498.760066:0:14663:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d107f0000. 00000020:00000010:3.0:1773941498.760069:0:14663:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d1817da00. 00008000:00000010:3.0:1773941498.760072:0:14663:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff988d209194e0. 00000020:00000001:3.0:1773941498.760075:0:14663:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446630330235616504 : -113743473935112 : ffff988d08a3fcf8) 00000020:00001000:3.0:1773941498.760077:0:14663:0:(cl_object.c:831:cl_env_get()) 1@ffff988d08a3fcf0 00000008:00000001:3.0:1773941498.760080:0:14663:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:3.0:1773941498.760082:0:14663:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:3.0:1773941498.760084:0:14663:0:(osc_cache.c:2316:osc_next_obj()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1773941498.760085:0:14663:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000001:00000001:3.0:1773941498.760087:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941498.760088:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00010000:00000010:2.1:1773941498.760265:0:26:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d00030000. 00010000:00000010:2.1:1773941498.760273:0:26:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d21ec1800. 00000001:00000001:3.0:1773941498.760415:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:3.0:1773941498.760417:0:14663:0:(cl_object.c:925:cl_env_put()) 1@ffff988d08a3fcf0 00000020:00000010:3.0:1773941498.760420:0:14663:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d1817d800. 00000020:00000010:3.0:1773941498.760424:0:14663:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d111f7780. 00008000:00000010:3.0:1773941498.760427:0:14663:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d107f0000. 00000020:00000010:3.0:1773941498.760429:0:14663:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d1817c600. 00008000:00000010:3.0:1773941498.760431:0:14663:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff988d209194e0. 00000020:00000010:3.0:1773941498.760433:0:14663:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d1817da00. 00000020:00000010:3.0:1773941498.760435:0:14663:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff988d08a3fcf0. 00000001:00000001:3.0:1773941498.760437:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941498.760438:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941498.760750:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941498.760758:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941498.760759:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941498.761110:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:3.0:1773941498.761113:0:14663:0:(mdc_request.c:2936:mdc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941498.761114:0:14663:0:(obd_class.h:1226:obd_import_event()) Process leaving 00010000:00000001:3.0:1773941498.761117:0:14663:0:(ldlm_lib.c:818:client_disconnect_export()) Process leaving 00000020:00000001:3.0:1773941498.761119:0:14663:0:(genops.c:1462:class_disconnect()) Process entered 00000020:00000080:3.0:1773941498.761121:0:14663:0:(genops.c:1487:class_disconnect()) disconnect: cookie 0x8e86767622e28aa 00000020:00000040:3.0:1773941498.761124:0:14663:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d035c0800 with handle 0x8e86767622e28aa from hash 00000020:00000001:3.0:1773941498.761128:0:14663:0:(genops.c:1934:obd_stale_export_put()) Process entered 00000020:00000040:3.0:1773941498.761130:0:14663:0:(genops.c:898:class_export_put()) PUTting export ffff988d035c0800 : new refcount 1 00000020:00000001:3.0:1773941498.761132:0:14663:0:(genops.c:1959:obd_stale_export_put()) Process leaving 00000020:00000040:3.0:1773941498.761133:0:14663:0:(genops.c:898:class_export_put()) PUTting export ffff988d035c0800 : new refcount 0 00000020:00000080:3.0:1773941498.761135:0:14663:0:(genops.c:904:class_export_put()) final put ffff988d035c0800/e9ee4b2c-2bac-44e1-850a-8bf1479379ac 00000020:00000001:3.0:1773941498.761137:0:14663:0:(lprocfs_status_server.c:686:lprocfs_exp_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941498.761147:0:14663:0:(genops.c:1493:class_disconnect()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941498.761149:0:14663:0:(ldlm_lib.c:831:client_disconnect_export()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:3.0:1773941498.761151:0:14663:0:(obd_class.h:914:obd_disconnect()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:3.0:1773941498.761153:0:14663:0:(lmv_obd.c:579:lmv_disconnect_mdc()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:3.0:1773941498.761156:0:14663:0:(lmv_obd.c:534:lmv_disconnect_mdc()) Process entered 00020000:00000001:3.0:1773941498.761166:0:14663:0:(lu_tgt_descs.c:164:lu_qos_del_tgt()) Process entered 00020000:00001000:3.0:1773941498.761168:0:14663:0:(lu_tgt_descs.c:175:lu_qos_del_tgt()) removing server 192.168.204.114@tcp 00020000:00000010:3.0:1773941498.761170:0:14663:0:(lu_tgt_descs.c:177:lu_qos_del_tgt()) kfreed 'svr': 104 at ffff988d03580580. 00020000:00000001:3.0:1773941498.761173:0:14663:0:(lu_tgt_descs.c:186:lu_qos_del_tgt()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941498.761173:0:8:0:(genops.c:857:class_export_destroy()) Process entered 80000000:00000001:3.0:1773941498.761178:0:14663:0:(fld_request.c:185:fld_client_del_target()) Process entered 00000020:00000040:3.0:1773941498.761180:0:14663:0:(genops.c:898:class_export_put()) PUTting export ffff988d035c0000 : new refcount 3 00000020:00000080:0.0:1773941498.761180:0:8:0:(genops.c:863:class_export_destroy()) destroying export ffff988d035c0800/e9ee4b2c-2bac-44e1-850a-8bf1479379ac for lustre-MDT0000-mdc-ffff988d08f0a000 80000000:00000010:3.0:1773941498.761184:0:14663:0:(fld_request.c:197:fld_client_del_target()) kfreed 'target': 40 at ffff988d03ec7540. 80000000:00000001:3.0:1773941498.761188:0:14663:0:(fld_request.c:198:fld_client_del_target()) Process leaving (rc=0 : 0 : 0) 40000000:00000001:3.0:1773941498.761190:0:14663:0:(fid_request.c:460:client_fid_fini()) Process entered 00000020:00000040:0.0:1773941498.761190:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 8 to 192.168.204.114@tcp 40000000:00000001:3.0:1773941498.761193:0:14663:0:(fid_request.c:376:seq_client_fini()) Process entered 00000020:00000001:0.0:1773941498.761196:0:8:0:(obd_class.h:1017:obd_destroy_export()) Process entered 00000020:00000001:0.0:1773941498.761200:0:8:0:(obd_class.h:1021:obd_destroy_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1773941498.761203:0:8:0:(obd_config.c:1058:class_decref()) Decref lustre-MDT0000-mdc-ffff988d08f0a000 (ffff988d089ab370) now 5 - export 00000020:00000010:0.0:1773941498.761207:0:8:0:(genops.c:879:class_export_destroy()) kfreed_rcu 'exp': 1208 at ffff988d035c0800. 00000020:00000001:0.0:1773941498.761228:0:8:0:(genops.c:880:class_export_destroy()) Process leaving 00000020:00000040:3.0:1773941498.761250:0:14663:0:(genops.c:898:class_export_put()) PUTting export ffff988d035c0000 : new refcount 2 40000000:00000001:3.0:1773941498.761252:0:14663:0:(fid_request.c:386:seq_client_fini()) Process leaving 40000000:00000010:3.0:1773941498.761254:0:14663:0:(fid_request.c:465:client_fid_fini()) kfreed 'cli->cl_seq': 224 at ffff988d21fada00. 40000000:00000001:3.0:1773941498.761257:0:14663:0:(fid_request.c:470:client_fid_fini()) Process leaving (rc=0 : 0 : 0) 00800000:00000040:3.0:1773941498.761259:0:14663:0:(lmv_obd.c:568:lmv_disconnect_mdc()) Disconnected from lustre-MDT0001-mdc-ffff988d08f0a000(e9ee4b2c-2bac-44e1-850a-8bf1479379ac) successfully 00800000:00000001:3.0:1773941498.761261:0:14663:0:(obd_class.h:1347:obd_register_observer()) Process entered 00800000:00000001:3.0:1773941498.761267:0:14663:0:(obd_class.h:1360:obd_register_observer()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:3.0:1773941498.761268:0:14663:0:(obd_class.h:902:obd_disconnect()) Process entered 00010000:00000001:3.0:1773941498.761270:0:14663:0:(ldlm_lib.c:759:client_disconnect_export()) Process entered 00010000:00000040:3.0:1773941498.761272:0:14663:0:(ldlm_lib.c:772:client_disconnect_export()) disconnect lustre-MDT0001-mdc-ffff988d08f0a000 - 1 00000100:00000001:3.0:1773941498.761274:0:14663:0:(pinger.c:388:ptlrpc_pinger_del_import()) Process entered 00000100:00080000:3.0:1773941498.761276:0:14663:0:(pinger.c:396:ptlrpc_pinger_del_import()) removing pingable import e9ee4b2c-2bac-44e1-850a-8bf1479379ac->lustre-MDT0001_UUID 00000020:00000001:3.0:1773941498.761278:0:14663:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941498.761280:0:14663:0:(genops.c:1242:class_import_put()) import ffff988d035c6000 refcount=2 obd=lustre-MDT0001-mdc-ffff988d08f0a000 00000020:00000001:3.0:1773941498.761281:0:14663:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941498.761282:0:14663:0:(pinger.c:401:ptlrpc_pinger_del_import()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941498.761284:0:14663:0:(ldlm_request.c:2367:ldlm_cli_cancel_unused()) Process entered 00000001:00000001:3.0:1773941498.761285:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941498.761286:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941498.761636:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941498.761638:0:14663:0:(ldlm_request.c:2379:ldlm_cli_cancel_unused()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941498.761640:0:14663:0:(ldlm_resource.c:1268:ldlm_namespace_free_prior()) Process entered 00010000:00000001:3.0:1773941498.761643:0:14663:0:(ldlm_resource.c:1207:__ldlm_namespace_free()) Process entered 00000001:00000001:3.0:1773941498.761644:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941498.761645:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941498.761997:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941498.761999:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941498.762000:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941498.762324:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941498.762327:0:14663:0:(ldlm_resource.c:1251:__ldlm_namespace_free()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941498.762328:0:14663:0:(ldlm_resource.c:1293:ldlm_namespace_free_prior()) Process leaving 00000100:00000001:3.0:1773941498.762330:0:14663:0:(import.c:1924:ptlrpc_disconnect_import()) Process entered 00000100:00000001:3.0:1773941498.762332:0:14663:0:(import.c:1858:ptlrpc_disconnect_import_async()) Process entered 00000100:00000001:3.0:1773941498.762334:0:14663:0:(import.c:1750:ptlrpc_disconnect_prep_req()) Process entered 00000100:00000010:3.0:1773941498.762338:0:14663:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d111f0a80. 00000020:00000040:3.0:1773941498.762341:0:14663:0:(genops.c:1229:class_import_get()) import ffff988d035c6000 refcount=3 obd=lustre-MDT0001-mdc-ffff988d08f0a000 00000100:00000001:3.0:1773941498.762344:0:14663:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941498.762347:0:14663:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941498.762349:0:14663:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941498.762354:0:14663:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff988d1a2ec200. 00000100:00000001:3.0:1773941498.762358:0:14663:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941498.762360:0:14663:0:(import.c:1788:ptlrpc_disconnect_prep_req()) Process leaving (rc=18446630330377898624 : -113743331652992 : ffff988d111f0a80) 00000100:00080000:3.0:1773941498.762362:0:14663:0:(import.c:65:import_set_state_nolock()) ffff988d035c6000 lustre-MDT0001_UUID: changing import state from FULL to CONNECTING 00000100:00000001:3.0:1773941498.762371:0:14663:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941498.762373:0:14663:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941498.762379:0:14663:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b933c3a] to pc [ptlrpcd_rcv+-1] req@ffff988d111f0a80 x1860112291476352/t0(0) o39->lustre-MDT0001-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941498.762394:0:14663:0:(import.c:1904:ptlrpc_disconnect_import_async()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.762429:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941498.762433:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941498.762435:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941498.762443:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d111f0a80 x1860112291476352/t0(0) o39->lustre-MDT0001-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941498.762454:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941498.762457:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941498.762460:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941498.762464:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941498.762467:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941498.762471:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d111f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9ee4b2c-2bac-44e1-850a-8bf1479379ac:2400:1860112291476352:192.168.204.114@tcp:39:umount.0 00000100:00000001:2.0:1773941498.762476:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941498.762480:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941498.762482:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941498.762484:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941498.762489:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff988d0350f000. 02000000:00000001:2.0:1773941498.762491:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941498.762497:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941498.762502:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e9a0. 00000400:00000010:2.0:1773941498.762507:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045167f8. 00000100:00000200:2.0:1773941498.762513:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112291476352, portal 10 00000100:00000001:2.0:1773941498.762515:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941498.762517:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330377898624 : -113743331652992 : ffff988d111f0a80) 00000100:00000040:2.0:1773941498.762525:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d111f0a80 x1860112291476352/t0(0) o39->lustre-MDT0001-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941508 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941498.762534:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941498.762538:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:2.0:1773941498.762542:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516660. 00000100:00000200:2.0:1773941498.762546:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 12, xid 1860112291476352, offset 0 00000400:00000200:2.0:1773941498.762553:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941498.762560:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941498.762567:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941498.762579:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941498.762584:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941498.762587:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941498.762591:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:2.0:1773941498.762596:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [109:109:256:4294967295] 192.168.204.114@tcp LPNI seq info [109:109:8:4294967295] 00000400:00000200:2.0:1773941498.762601:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941498.762614:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:2.0:1773941498.762620:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:2.0:1773941498.762625:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d217a8b00. 00000800:00000200:2.0:1773941498.762632:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:2.0:1773941498.762639:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:2.0:1773941498.762643:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d217a8b00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1773941498.762659:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941498.762664:0:2400:0:(niobuf.c:634:kick_cpu_latency()) lustre-MDT0001-mdc-ffff988d08f0a000: using a timeout of 3973 usecs (4 jiffies) 00000400:00000200:2.0:1773941498.762668:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941498.762671:0:2400:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff988d08f0a000: PM QoS new deadline estimation for cpu 0 is 4294889107 00000100:00000001:2.0:1773941498.762676:0:2400:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff988d08f0a000: PM QoS request active for cpu 0, keep current deadline 4294890340 00000100:00000001:2.0:1773941498.762679:0:2400:0:(niobuf.c:662:kick_cpu_latency()) lustre-MDT0001-mdc-ffff988d08f0a000: PM QoS new deadline estimation for cpu 1 is 4294889107 00000100:00000001:2.0:1773941498.762682:0:2400:0:(niobuf.c:691:kick_cpu_latency()) lustre-MDT0001-mdc-ffff988d08f0a000: PM QoS request active for cpu 1, keep current deadline 4294890340 00000100:00000001:2.0:1773941498.762685:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941498.762688:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.762692:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.762694:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941498.762700:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941498.762702:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941498.762705:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941498.762706:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941498.762708:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.762711:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941498.762767:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941498.762772:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d217a8b00. 00000400:00000200:0.0:1773941498.762777:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941498.762782:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941498.762786:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516660 00000400:00000010:0.0:1773941498.762789:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516660. 00000100:00000001:0.0:1773941498.762794:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941498.762800:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d111f0a80 x1860112291476352/t0(0) o39->lustre-MDT0001-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941508 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941498.762810:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941498.762815:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d111f0a80 x1860112291476352/t0(0) o39->lustre-MDT0001-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941508 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941498.762837:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.762840:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941498.764383:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941498.764396:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941498.764399:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941498.764403:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941498.764408:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941498.764415:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941498.764422:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941498.764427:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941498.764431:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941498.764436:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 10 MB=0x69bc324603380 00000400:00000200:0.0:1773941498.764441:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941498.764447:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index a from 12345-192.168.204.114@tcp of length 224/224 into md 0x8b1 [1] + 224 00000800:00000001:0.0:1773941498.764453:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941498.764472:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941498.764475:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941498.764480:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941498.764484:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941498.764491:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d111f0a80 x1860112291476352/t0(0) o39->lustre-MDT0001-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941508 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941498.764502:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff988d111f0a80 x1860112291476352/t0(0) o39->lustre-MDT0001-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941508 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941498.764519:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941498.764528:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941498.764533:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941498.764540:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941498.764543:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941498.764548:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941498.764550:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941498.764553:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e9a0. 00000400:00000200:1.0:1773941498.764557:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045167f8 00000400:00000010:1.0:1773941498.764558:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045167f8. 00000100:00000001:1.0:1773941498.764560:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941498.764565:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d111f0a80 x1860112291476352/t0(0) o39->lustre-MDT0001-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941508 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941498.764585:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d111f0a80 x1860112291476352/t0(0) o39->lustre-MDT0001-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941508 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.764590:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941498.764591:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941498.764594:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941498.764596:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941498.764597:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941498.764599:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941498.764601:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941498.764607:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d035c6458 time=1 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941498.764609:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d035c6410 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941498.764612:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1773941498.764613:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941498.764615:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00080000:1.0:1773941498.764619:0:2400:0:(ldlm_request.c:1534:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff988d111f0a80 x1860112291476352/t0(0) o39->lustre-MDT0001-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941508 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'umount.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941498.764625:0:2400:0:(ldlm_request.c:1535:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.764627:0:2400:0:(client.c:1532:ptlrpc_save_versions()) Process entered 00000100:00000040:1.0:1773941498.764628:0:2400:0:(client.c:1539:ptlrpc_save_versions()) Client save versions [0x0/0x0] 00000100:00000001:1.0:1773941498.764630:0:2400:0:(client.c:1541:ptlrpc_save_versions()) Process leaving 00000100:00000001:1.0:1773941498.764631:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941498.764632:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330377898624 : -113743331652992 : ffff988d111f0a80) 00000100:00000001:1.0:1773941498.764635:0:2400:0:(client.c:3065:ptlrpc_free_committed()) Process entered 00000100:00000040:1.0:1773941498.764636:0:2400:0:(client.c:3077:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff988d08f0a000: skip recheck: last_committed 0 00000100:00000001:1.0:1773941498.764637:0:2400:0:(client.c:3078:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1773941498.764639:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941498.764642:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d111f0a80 x1860112291476352/t8589934593(8589934593) o39->lustre-MDT0001-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941508 ref 2 fl Rpc:RNQU/204/0 rc 0/0 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.764647:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941498.764649:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941498.764650:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1773941498.764652:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d035c6000 lustre-MDT0001_UUID: changing import state from CONNECTING to CLOSED 00000100:00000001:1.0:1773941498.764659:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 00000002:00000001:1.0:1773941498.764661:0:2400:0:(mdc_request.c:2862:mdc_import_event()) Process entered 00000002:00000001:1.0:1773941498.764663:0:2400:0:(mdc_request.c:2936:mdc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.764664:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941498.764665:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 00000002:00000001:1.0:1773941498.764666:0:2400:0:(mdc_request.c:2862:mdc_import_event()) Process entered 00000002:00000001:1.0:1773941498.764668:0:2400:0:(mdc_request.c:2936:mdc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.764669:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000040:1.0:1773941498.764680:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d111f0a80 x1860112291476352/t8589934593(8589934593) o39->lustre-MDT0001-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941508 ref 2 fl Interpret:RNQU/204/0 rc 0/0 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941498.764686:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d111f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e9ee4b2c-2bac-44e1-850a-8bf1479379ac:2400:1860112291476352:192.168.204.114@tcp:39:umount.0 00000100:00000001:1.0:1773941498.764691:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941498.764693:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941498.764695:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d111f0a80 x1860112291476352/t8589934593(8589934593) o39->lustre-MDT0001-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941508 ref 2 fl Complete:RNQU/204/0 rc 0/0 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941498.764697:0:14663:0:(import.c:1967:ptlrpc_disconnect_import()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.764702:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941498.764704:0:14663:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00000001:1.0:1773941498.764705:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1773941498.764707:0:14663:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-MDT0001_UUID INVALID 00000100:00000001:1.0:1773941498.764710:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941498.764711:0:14663:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00000001:1.0:1773941498.764714:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941498.764715:0:14663:0:(client.c:3065:ptlrpc_free_committed()) Process entered 00000100:00100000:3.0:1773941498.764718:0:14663:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff988d08f0a000: committing for last_committed 0 gen 2 00000100:00000001:1.0:1773941498.764719:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941498.764722:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941498.764726:0:14663:0:(client.c:3119:ptlrpc_free_committed()) @@@ free request with old gen req@ffff988d111f0a80 x1860112291476352/t8589934593(8589934593) o39->lustre-MDT0001-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941508 ref 1 fl Complete:RNQU/204/0 rc 0/0 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941498.764732:0:14663:0:(client.c:3120:ptlrpc_free_committed()) Process leaving via free_req (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941498.764735:0:14663:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941498.764738:0:14663:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d111f0a80 x1860112291476352/t8589934593(8589934593) o39->lustre-MDT0001-mdc-ffff988d08f0a000@192.168.204.114@tcp:12/10 lens 224/224 e 0 to 0 dl 1773941508 ref 1 fl Complete:RNQU/204/0 rc 0/0 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941498.764743:0:14663:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941498.764744:0:14663:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941498.764746:0:14663:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff988d0350f000. 02000000:00000001:3.0:1773941498.764749:0:14663:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941498.764750:0:14663:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941498.764751:0:14663:0:(genops.c:1242:class_import_put()) import ffff988d035c6000 refcount=2 obd=lustre-MDT0001-mdc-ffff988d08f0a000 00000020:00000001:3.0:1773941498.764753:0:14663:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941498.764755:0:14663:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff988d1a2ec200. 02000000:00000001:3.0:1773941498.764757:0:14663:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941498.764758:0:14663:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941498.764760:0:14663:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d111f0a80. 00000100:00000001:3.0:1773941498.764762:0:14663:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941498.764763:0:14663:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941498.764765:0:14663:0:(client.c:3191:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff988d08f0a000: normal: skip=0 replay=1/1 committed=0/0 00000100:00000001:3.0:1773941498.764767:0:14663:0:(client.c:3192:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1773941498.764773:0:14663:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941498.764774:0:14663:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941498.764776:0:14663:0:(obd_class.h:1215:obd_import_event()) Process entered 00000002:00000001:3.0:1773941498.764777:0:14663:0:(mdc_request.c:2862:mdc_import_event()) Process entered 00000002:00000001:3.0:1773941498.764779:0:14663:0:(mdc_request.c:2936:mdc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941498.764780:0:14663:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00100000:3.0:1773941498.764782:0:14663:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:3.0:1773941498.764784:0:14663:0:(obd_class.h:1215:obd_import_event()) Process entered 00000002:00000001:3.0:1773941498.764784:0:14663:0:(mdc_request.c:2862:mdc_import_event()) Process entered 00000001:00000001:3.0:1773941498.764786:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941498.764787:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941498.765168:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941498.765170:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941498.765171:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941498.765594:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941498.765598:0:14663:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:3.0:1773941498.765603:0:14663:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff988d08a3fcf0. 00000020:00000010:3.0:1773941498.765609:0:14663:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d1817da00. 00000020:00000010:3.0:1773941498.765613:0:14663:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d1817c600. 00000020:00000010:3.0:1773941498.765617:0:14663:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d111f7780. 00008000:00000010:3.0:1773941498.765623:0:14663:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d107f0000. 00000020:00000010:3.0:1773941498.765629:0:14663:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d1817d800. 00008000:00000010:3.0:1773941498.765633:0:14663:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff988d209194e0. 00000020:00000001:3.0:1773941498.765637:0:14663:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446630330235616504 : -113743473935112 : ffff988d08a3fcf8) 00000020:00001000:3.0:1773941498.765640:0:14663:0:(cl_object.c:831:cl_env_get()) 1@ffff988d08a3fcf0 00000008:00000001:3.0:1773941498.765643:0:14663:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:3.0:1773941498.765645:0:14663:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:3.0:1773941498.765648:0:14663:0:(osc_cache.c:2316:osc_next_obj()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1773941498.765650:0:14663:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000001:00000001:3.0:1773941498.765651:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941498.765653:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941498.766087:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:3.0:1773941498.766091:0:14663:0:(cl_object.c:925:cl_env_put()) 1@ffff988d08a3fcf0 00000020:00000010:3.0:1773941498.766095:0:14663:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d1817c600. 00000020:00000010:3.0:1773941498.766099:0:14663:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d111f7780. 00008000:00000010:3.0:1773941498.766104:0:14663:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d107f0000. 00000020:00000010:3.0:1773941498.766109:0:14663:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d1817da00. 00008000:00000010:3.0:1773941498.766113:0:14663:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff988d209194e0. 00000020:00000010:3.0:1773941498.766124:0:14663:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d1817d800. 00000020:00000010:3.0:1773941498.766127:0:14663:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff988d08a3fcf0. 00000001:00000001:3.0:1773941498.766131:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941498.766133:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941498.766573:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941498.766577:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941498.766578:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941498.767009:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:3.0:1773941498.767012:0:14663:0:(mdc_request.c:2936:mdc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941498.767014:0:14663:0:(obd_class.h:1226:obd_import_event()) Process leaving 00010000:00000001:3.0:1773941498.767019:0:14663:0:(ldlm_lib.c:818:client_disconnect_export()) Process leaving 00000020:00000001:3.0:1773941498.767021:0:14663:0:(genops.c:1462:class_disconnect()) Process entered 00000020:00000080:3.0:1773941498.767024:0:14663:0:(genops.c:1487:class_disconnect()) disconnect: cookie 0x8e86767622e28b1 00000020:00000040:3.0:1773941498.767028:0:14663:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d035c0000 with handle 0x8e86767622e28b1 from hash 00000020:00000001:3.0:1773941498.767034:0:14663:0:(genops.c:1934:obd_stale_export_put()) Process entered 00000020:00000040:3.0:1773941498.767037:0:14663:0:(genops.c:898:class_export_put()) PUTting export ffff988d035c0000 : new refcount 1 00000020:00000001:3.0:1773941498.767039:0:14663:0:(genops.c:1959:obd_stale_export_put()) Process leaving 00000020:00000040:3.0:1773941498.767041:0:14663:0:(genops.c:898:class_export_put()) PUTting export ffff988d035c0000 : new refcount 0 00000020:00000080:3.0:1773941498.767043:0:14663:0:(genops.c:904:class_export_put()) final put ffff988d035c0000/e9ee4b2c-2bac-44e1-850a-8bf1479379ac 00000020:00000001:3.0:1773941498.767046:0:14663:0:(lprocfs_status_server.c:686:lprocfs_exp_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941498.767061:0:14663:0:(genops.c:1493:class_disconnect()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941498.767064:0:14663:0:(ldlm_lib.c:831:client_disconnect_export()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:3.0:1773941498.767068:0:14663:0:(obd_class.h:914:obd_disconnect()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:3.0:1773941498.767071:0:14663:0:(lmv_obd.c:579:lmv_disconnect_mdc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941498.767075:0:8:0:(genops.c:857:class_export_destroy()) Process entered 00000020:00000080:0.0:1773941498.767079:0:8:0:(genops.c:863:class_export_destroy()) destroying export ffff988d035c0000/e9ee4b2c-2bac-44e1-850a-8bf1479379ac for lustre-MDT0001-mdc-ffff988d08f0a000 00000020:00000040:0.0:1773941498.767084:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 7 to 192.168.204.114@tcp 00000020:00000001:0.0:1773941498.767087:0:8:0:(obd_class.h:1017:obd_destroy_export()) Process entered 00000020:00000001:3.0:1773941498.767091:0:14663:0:(genops.c:1462:class_disconnect()) Process entered 00000020:00000080:3.0:1773941498.767094:0:14663:0:(genops.c:1487:class_disconnect()) disconnect: cookie 0x8e86767622e28a3 00000020:00000001:0.0:1773941498.767096:0:8:0:(obd_class.h:1021:obd_destroy_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1773941498.767099:0:14663:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d035c1000 with handle 0x8e86767622e28a3 from hash 00000020:00000040:0.0:1773941498.767100:0:8:0:(obd_config.c:1058:class_decref()) Decref lustre-MDT0001-mdc-ffff988d08f0a000 (ffff988d089acd28) now 5 - export 00000020:00000001:3.0:1773941498.767106:0:14663:0:(genops.c:1934:obd_stale_export_put()) Process entered 00000020:00000010:0.0:1773941498.767106:0:8:0:(genops.c:879:class_export_destroy()) kfreed_rcu 'exp': 1208 at ffff988d035c0000. 00000020:00000040:3.0:1773941498.767110:0:14663:0:(genops.c:898:class_export_put()) PUTting export ffff988d035c1000 : new refcount 1 00000020:00000001:0.0:1773941498.767112:0:8:0:(genops.c:880:class_export_destroy()) Process leaving 00000020:00000001:3.0:1773941498.767113:0:14663:0:(genops.c:1959:obd_stale_export_put()) Process leaving 00000020:00000040:3.0:1773941498.767116:0:14663:0:(genops.c:898:class_export_put()) PUTting export ffff988d035c1000 : new refcount 0 00000020:00000080:3.0:1773941498.767119:0:14663:0:(genops.c:904:class_export_put()) final put ffff988d035c1000/e9ee4b2c-2bac-44e1-850a-8bf1479379ac 00000020:00000001:3.0:1773941498.767122:0:14663:0:(lprocfs_status_server.c:686:lprocfs_exp_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941498.767132:0:14663:0:(genops.c:1493:class_disconnect()) Process leaving (rc=0 : 0 : 0) 00800000:00000001:3.0:1773941498.767134:0:14663:0:(lmv_obd.c:600:lmv_disconnect()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:3.0:1773941498.767139:0:14663:0:(obd_class.h:914:obd_disconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941498.767139:0:8:0:(genops.c:857:class_export_destroy()) Process entered 00000020:00000080:0.0:1773941498.767142:0:8:0:(genops.c:863:class_export_destroy()) destroying export ffff988d035c1000/e9ee4b2c-2bac-44e1-850a-8bf1479379ac for lustre-clilmv-ffff988d08f0a000 00000020:00000001:0.0:1773941498.767144:0:8:0:(obd_class.h:1017:obd_destroy_export()) Process entered 00000020:00000001:0.0:1773941498.767145:0:8:0:(obd_class.h:1021:obd_destroy_export()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:3.0:1773941498.767151:0:14663:0:(llite_lib.c:995:client_common_put_super()) Process leaving 00000020:00000040:0.0:1773941498.767151:0:8:0:(obd_config.c:1058:class_decref()) Decref lustre-clilmv-ffff988d08f0a000 (ffff988d089a8000) now 4 - export 00000020:00000010:0.0:1773941498.767154:0:8:0:(genops.c:879:class_export_destroy()) kfreed_rcu 'exp': 1208 at ffff988d035c1000. 00000020:00000001:3.0:1773941498.767157:0:14663:0:(obd_config.c:2489:class_manual_cleanup()) Process entered 00000020:01000000:3.0:1773941498.767161:0:14663:0:(obd_config.c:2502:class_manual_cleanup()) Manual cleanup of lustre-clilov-ffff988d08f0a000 (flags='') 00000020:00000001:0.0:1773941498.767162:0:8:0:(genops.c:880:class_export_destroy()) Process leaving 00000020:00000010:3.0:1773941498.767166:0:14663:0:(obd_config.c:2506:class_manual_cleanup()) kmalloced '(lcfg)': 80 at ffff988d111f7780. 00000020:00000080:3.0:1773941498.767171:0:14663:0:(obd_config.c:1506:class_process_config()) processing cmd: cf004 00000020:00000001:3.0:1773941498.767175:0:14663:0:(obd_config.c:936:class_cleanup()) Process entered 00000020:00000080:3.0:1773941498.767178:0:14663:0:(obd_config.c:991:class_cleanup()) lustre-clilov-ffff988d08f0a000: forcing exports to disconnect: 0/1 00000020:00000001:3.0:1773941498.767182:0:14663:0:(genops.c:1562:class_disconnect_exports()) Process entered 00000020:00080000:3.0:1773941498.767184:0:14663:0:(genops.c:1577:class_disconnect_exports()) OBD device 1 (ffff988d089a99b8) has no exports 00000020:00000001:3.0:1773941498.767187:0:14663:0:(genops.c:1578:class_disconnect_exports()) Process leaving 00000020:00000001:3.0:1773941498.767190:0:14663:0:(obd_class.h:604:obd_precleanup()) Process entered 00000020:00000001:3.0:1773941498.767192:0:14663:0:(obd_class.h:609:obd_precleanup()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941498.767210:0:14663:0:(hash.c:1113:cfs_hash_destroy()) Process entered 00000001:00000010:3.0:1773941498.767233:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff988d0335da00. 00000001:00000010:3.0:1773941498.767237:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff988d0335cc00. 00000001:00000010:3.0:1773941498.767240:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff988d0335c600. 00000001:00000010:3.0:1773941498.767242:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff988d0335ca00. 00000001:00000010:3.0:1773941498.767245:0:14663:0:(hash.c:913:cfs_hash_buckets_free()) kfreed 'buckets': 32 at ffff988d107cf0e0. 00000001:00000010:3.0:1773941498.767248:0:14663:0:(hash.c:1163:cfs_hash_destroy()) kfreed 'hs': 232 at ffff988d02cbac00. 00000001:00000001:3.0:1773941498.767251:0:14663:0:(hash.c:1165:cfs_hash_destroy()) Process leaving 00000020:00000040:3.0:1773941498.767253:0:14663:0:(obd_config.c:1058:class_decref()) Decref lustre-clilov-ffff988d08f0a000 (ffff988d089a99b8) now 3 - setup 00000020:00000001:3.0:1773941498.767256:0:14663:0:(obd_config.c:1021:class_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941498.767258:0:14663:0:(obd_config.c:1659:class_process_config()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000080:3.0:1773941498.767261:0:14663:0:(obd_config.c:1506:class_process_config()) processing cmd: cf002 00000020:00000001:3.0:1773941498.767264:0:14663:0:(obd_config.c:899:class_detach()) Process entered 00000020:00000040:3.0:1773941498.767269:0:14663:0:(obd_config.c:1058:class_decref()) Decref lustre-clilov-ffff988d08f0a000 (ffff988d089a99b8) now 2 - obd_device_list 00000020:00000080:3.0:1773941498.767272:0:14663:0:(obd_config.c:919:class_detach()) detach on obd lustre-clilov-ffff988d08f0a000 (uuid e9ee4b2c-2bac-44e1-850a-8bf1479379ac) 00000020:00000040:3.0:1773941498.767274:0:14663:0:(obd_config.c:1058:class_decref()) Decref lustre-clilov-ffff988d08f0a000 (ffff988d089a99b8) now 1 - newdev 00000020:00000040:3.0:1773941498.767277:0:14663:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d035c6800 with handle 0x8e86767622e2872 from hash 00000020:00000040:3.0:1773941498.767281:0:14663:0:(genops.c:898:class_export_put()) PUTting export ffff988d035c6800 : new refcount 0 00000020:00000080:3.0:1773941498.767283:0:14663:0:(genops.c:904:class_export_put()) final put ffff988d035c6800/e9ee4b2c-2bac-44e1-850a-8bf1479379ac 00000020:00000001:3.0:1773941498.767285:0:14663:0:(lprocfs_status_server.c:686:lprocfs_exp_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941498.767288:0:14663:0:(genops.c:857:class_export_destroy()) Process entered 00000020:00000080:3.0:1773941498.767289:0:14663:0:(genops.c:863:class_export_destroy()) destroying export ffff988d035c6800/e9ee4b2c-2bac-44e1-850a-8bf1479379ac for lustre-clilov-ffff988d08f0a000 00000020:00000001:3.0:1773941498.767299:0:14663:0:(obd_class.h:1017:obd_destroy_export()) Process entered 00000020:00000001:3.0:1773941498.767301:0:14663:0:(obd_class.h:1021:obd_destroy_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941498.767303:0:14663:0:(genops.c:879:class_export_destroy()) kfreed_rcu 'exp': 1208 at ffff988d035c6800. 00000020:00000001:3.0:1773941498.767310:0:14663:0:(genops.c:880:class_export_destroy()) Process leaving 00000020:00000040:3.0:1773941498.767313:0:14663:0:(genops.c:437:class_free_dev()) Release obd device lustre-clilov-ffff988d08f0a000 obd_type name = lov 00000020:01000000:3.0:1773941498.767315:0:14663:0:(genops.c:440:class_free_dev()) finishing cleanup of obd lustre-clilov-ffff988d08f0a000 (e9ee4b2c-2bac-44e1-850a-8bf1479379ac) 00000020:00000001:3.0:1773941498.767317:0:14663:0:(obd_class.h:633:obd_cleanup()) Process entered 00000020:00000001:3.0:1773941498.767319:0:14663:0:(obd_class.h:638:obd_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941498.767321:0:14663:0:(genops.c:66:obd_device_free()) slab-freed '(obd)': 6584 at ffff988d089a99b8. 00000020:00000001:3.0:1773941498.767328:0:14663:0:(obd_config.c:923:class_detach()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941498.767331:0:14663:0:(obd_config.c:1655:class_process_config()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1773941498.767334:0:14663:0:(obd_config.c:2523:class_manual_cleanup()) kfreed 'lcfg': 80 at ffff988d111f7780. 00000020:00000001:3.0:1773941498.767337:0:14663:0:(obd_config.c:2524:class_manual_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941498.767340:0:14663:0:(obd_config.c:2489:class_manual_cleanup()) Process entered 00000020:01000000:3.0:1773941498.767342:0:14663:0:(obd_config.c:2502:class_manual_cleanup()) Manual cleanup of lustre-clilmv-ffff988d08f0a000 (flags='') 00000020:00000010:3.0:1773941498.767345:0:14663:0:(obd_config.c:2506:class_manual_cleanup()) kmalloced '(lcfg)': 80 at ffff988d111f7780. 00000020:00000080:3.0:1773941498.767347:0:14663:0:(obd_config.c:1506:class_process_config()) processing cmd: cf004 00000020:00000001:3.0:1773941498.767349:0:14663:0:(obd_config.c:936:class_cleanup()) Process entered 00000020:00000080:3.0:1773941498.767351:0:14663:0:(obd_config.c:991:class_cleanup()) lustre-clilmv-ffff988d08f0a000: forcing exports to disconnect: 0/1 00000020:00000001:3.0:1773941498.767353:0:14663:0:(genops.c:1562:class_disconnect_exports()) Process entered 00000020:00080000:3.0:1773941498.767355:0:14663:0:(genops.c:1577:class_disconnect_exports()) OBD device 2 (ffff988d089a8000) has no exports 00000020:00000001:3.0:1773941498.767357:0:14663:0:(genops.c:1578:class_disconnect_exports()) Process leaving 00000020:00000001:3.0:1773941498.767359:0:14663:0:(obd_class.h:604:obd_precleanup()) Process entered 00000020:00000010:3.0:1773941498.767362:0:14663:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d1817d800. 00000020:00000010:3.0:1773941498.767366:0:14663:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d1817da00. 00000040:00000010:3.0:1773941498.767370:0:14663:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d1817c600. 00000020:00000010:3.0:1773941498.767375:0:14663:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d07233000. 00000020:00000010:3.0:1773941498.767379:0:14663:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d1817d600. 00040000:00000010:3.0:1773941498.767384:0:14663:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d1a2ec200. 00040000:00000010:3.0:1773941498.767388:0:14663:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d1817d400. 00000020:00000010:3.0:1773941498.767393:0:14663:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff988d03ec7540. 00000020:00000001:3.0:1773941498.767399:0:14663:0:(kernelcomm.c:1231:libcfs_kkuc_group_rem()) Process entered 00000020:00000001:3.0:1773941498.767401:0:14663:0:(kernelcomm.c:1272:libcfs_kkuc_group_put()) Process entered 00000020:00000001:3.0:1773941498.767403:0:14663:0:(kernelcomm.c:1285:libcfs_kkuc_group_put()) Process leaving (rc=18446744073709551605 : -11 : fffffffffffffff5) 00000020:00000001:3.0:1773941498.767412:0:14663:0:(kernelcomm.c:1263:libcfs_kkuc_group_rem()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941498.767418:0:14663:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff988d03ec7540. 00000020:00000010:3.0:1773941498.767424:0:14663:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d1817da00. 00000040:00000010:3.0:1773941498.767429:0:14663:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d1817c600. 00000020:00000010:3.0:1773941498.767433:0:14663:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d07233000. 00000020:00000010:3.0:1773941498.767438:0:14663:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d1817d600. 00040000:00000010:3.0:1773941498.767442:0:14663:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d1a2ec200. 00040000:00000010:3.0:1773941498.767447:0:14663:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d1817d400. 00000020:00000010:3.0:1773941498.767451:0:14663:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d1817d800. 00000020:00000001:3.0:1773941498.767454:0:14663:0:(obd_class.h:623:obd_precleanup()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941498.767475:0:14663:0:(hash.c:1113:cfs_hash_destroy()) Process entered 00000001:00000010:3.0:1773941498.767481:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff988d065c5400. 00000001:00000010:3.0:1773941498.767485:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff988d0335d000. 00000001:00000010:3.0:1773941498.767489:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff988d0335dc00. 00000001:00000010:3.0:1773941498.767492:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff988d1817c400. 00000001:00000010:3.0:1773941498.767496:0:14663:0:(hash.c:913:cfs_hash_buckets_free()) kfreed 'buckets': 32 at ffff988d107cf360. 00000001:00000010:3.0:1773941498.767499:0:14663:0:(hash.c:1163:cfs_hash_destroy()) kfreed 'hs': 232 at ffff988d02cba300. 00000001:00000001:3.0:1773941498.767502:0:14663:0:(hash.c:1165:cfs_hash_destroy()) Process leaving 00000020:00000040:3.0:1773941498.767504:0:14663:0:(obd_config.c:1058:class_decref()) Decref lustre-clilmv-ffff988d08f0a000 (ffff988d089a8000) now 3 - setup 00000020:00000001:3.0:1773941498.767507:0:14663:0:(obd_config.c:1021:class_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941498.767509:0:14663:0:(obd_config.c:1659:class_process_config()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000080:3.0:1773941498.767511:0:14663:0:(obd_config.c:1506:class_process_config()) processing cmd: cf002 00000020:00000001:3.0:1773941498.767513:0:14663:0:(obd_config.c:899:class_detach()) Process entered 00000020:00000040:3.0:1773941498.767515:0:14663:0:(obd_config.c:1058:class_decref()) Decref lustre-clilmv-ffff988d08f0a000 (ffff988d089a8000) now 2 - obd_device_list 00000020:00000080:3.0:1773941498.767518:0:14663:0:(obd_config.c:919:class_detach()) detach on obd lustre-clilmv-ffff988d08f0a000 (uuid e9ee4b2c-2bac-44e1-850a-8bf1479379ac) 00000020:00000040:3.0:1773941498.767520:0:14663:0:(obd_config.c:1058:class_decref()) Decref lustre-clilmv-ffff988d08f0a000 (ffff988d089a8000) now 1 - newdev 00000020:00000040:3.0:1773941498.767524:0:14663:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d035c3000 with handle 0x8e86767622e2879 from hash 00000020:00000040:3.0:1773941498.767527:0:14663:0:(genops.c:898:class_export_put()) PUTting export ffff988d035c3000 : new refcount 0 00000020:00000080:3.0:1773941498.767529:0:14663:0:(genops.c:904:class_export_put()) final put ffff988d035c3000/e9ee4b2c-2bac-44e1-850a-8bf1479379ac 00000020:00000001:3.0:1773941498.767531:0:14663:0:(lprocfs_status_server.c:686:lprocfs_exp_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941498.767533:0:14663:0:(genops.c:857:class_export_destroy()) Process entered 00000020:00000080:3.0:1773941498.767535:0:14663:0:(genops.c:863:class_export_destroy()) destroying export ffff988d035c3000/e9ee4b2c-2bac-44e1-850a-8bf1479379ac for lustre-clilmv-ffff988d08f0a000 00000020:00000001:3.0:1773941498.767537:0:14663:0:(obd_class.h:1017:obd_destroy_export()) Process entered 00000020:00000001:3.0:1773941498.767545:0:14663:0:(obd_class.h:1021:obd_destroy_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941498.767547:0:14663:0:(genops.c:879:class_export_destroy()) kfreed_rcu 'exp': 1208 at ffff988d035c3000. 00000020:00000001:3.0:1773941498.767552:0:14663:0:(genops.c:880:class_export_destroy()) Process leaving 00000020:00000040:3.0:1773941498.767554:0:14663:0:(genops.c:437:class_free_dev()) Release obd device lustre-clilmv-ffff988d08f0a000 obd_type name = lmv 00000020:01000000:3.0:1773941498.767556:0:14663:0:(genops.c:440:class_free_dev()) finishing cleanup of obd lustre-clilmv-ffff988d08f0a000 (e9ee4b2c-2bac-44e1-850a-8bf1479379ac) 00000020:00000001:3.0:1773941498.767558:0:14663:0:(obd_class.h:633:obd_cleanup()) Process entered 00000020:00000010:3.0:1773941498.767561:0:14663:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d1817c400. 00000020:00000010:3.0:1773941498.767564:0:14663:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d1817d800. 00000040:00000010:3.0:1773941498.767567:0:14663:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d1817d400. 00000020:00000010:3.0:1773941498.767570:0:14663:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d07233000. 00000020:00000010:3.0:1773941498.767573:0:14663:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d1817d600. 00040000:00000010:3.0:1773941498.767576:0:14663:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d1a2ec200. 00040000:00000010:3.0:1773941498.767579:0:14663:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d1817c600. 00800000:00000001:3.0:1773941498.767583:0:14663:0:(lmv_obd.c:1313:lmv_device_free()) Process entered 00800000:00000010:3.0:1773941498.767586:0:14663:0:(lmv_obd.c:1319:lmv_device_free()) kfreed 'pat': 280 at ffff988d04ce5400. 00800000:00000010:3.0:1773941498.767590:0:14663:0:(lmv_obd.c:1319:lmv_device_free()) kfreed 'pat': 280 at ffff988d04ce5000. 80000000:00000001:3.0:1773941498.767594:0:14663:0:(fld_request.c:280:fld_client_fini()) Process entered 80000000:00000001:3.0:1773941498.767597:0:14663:0:(fld_cache.c:227:fld_cache_flush()) Process entered 80000000:00000001:3.0:1773941498.767599:0:14663:0:(fld_cache.c:198:fld_cache_shrink()) Process entered 80000000:00000010:3.0:1773941498.767602:0:14663:0:(fld_cache.c:96:fld_cache_entry_delete()) kfreed 'node': 56 at ffff988d03ec7e40. 80000000:00000040:3.0:1773941498.767605:0:14663:0:(fld_cache.c:217:fld_cache_shrink()) cli-lustre-clilmv-ffff988d08f0a000: FLD cache - Shrunk by 1 entries 80000000:00000001:3.0:1773941498.767608:0:14663:0:(fld_cache.c:219:fld_cache_shrink()) Process leaving (rc=0 : 0 : 0) 80000000:00000001:3.0:1773941498.767610:0:14663:0:(fld_cache.c:234:fld_cache_flush()) Process leaving 80000000:00000040:3.0:1773941498.767611:0:14663:0:(fld_cache.c:80:fld_cache_fini()) FLD cache statistics (cli-lustre-clilmv-ffff988d08f0a000): 80000000:00000040:3.0:1773941498.767613:0:14663:0:(fld_cache.c:81:fld_cache_fini()) Cache reqs: 5 80000000:00000040:3.0:1773941498.767616:0:14663:0:(fld_cache.c:82:fld_cache_fini()) Total reqs: 6 80000000:00000010:3.0:1773941498.767617:0:14663:0:(fld_cache.c:84:fld_cache_fini()) kfreed 'cache': 168 at ffff988d02eb6300. 80000000:00000001:3.0:1773941498.767621:0:14663:0:(fld_request.c:298:fld_client_fini()) Process leaving 00000020:00000010:3.0:1773941498.767804:0:14663:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 640 at ffff988d065f9c00. 00000020:00000010:3.0:1773941498.767810:0:14663:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 640 at ffff988d1118bc00. 00000020:00000010:3.0:1773941498.767814:0:14663:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 640 at ffff988d111d3000. 00000020:00000010:3.0:1773941498.767819:0:14663:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 512 at ffff988d04ce4400. 00000020:00000010:3.0:1773941498.767824:0:14663:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff988d065b1800. 00020000:00000001:3.0:1773941498.767829:0:14663:0:(lu_tgt_descs.c:164:lu_qos_del_tgt()) Process entered 00020000:00000001:3.0:1773941498.767832:0:14663:0:(lu_tgt_descs.c:169:lu_qos_del_tgt()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00020000:00000001:3.0:1773941498.767841:0:14663:0:(lu_tgt_descs.c:186:lu_qos_del_tgt()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00800000:00000010:3.0:1773941498.767845:0:14663:0:(lmv_obd.c:410:lmv_del_target()) kfreed 'tgt': 368 at ffff988d04ce5600. 00020000:00000001:3.0:1773941498.767849:0:14663:0:(lu_tgt_descs.c:164:lu_qos_del_tgt()) Process entered 00020000:00000001:3.0:1773941498.767851:0:14663:0:(lu_tgt_descs.c:169:lu_qos_del_tgt()) Process leaving via out (rc=18446744073709551614 : -2 : 0xfffffffffffffffe) 00020000:00000001:3.0:1773941498.767854:0:14663:0:(lu_tgt_descs.c:186:lu_qos_del_tgt()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00800000:00000010:3.0:1773941498.767857:0:14663:0:(lmv_obd.c:410:lmv_del_target()) kfreed 'tgt': 368 at ffff988d065dc400. 00020000:00000010:3.0:1773941498.767862:0:14663:0:(lu_tgt_descs.c:285:lu_tgt_descs_fini()) kfreed 'ltd->ltd_tgt_idx[i]': 4096 at ffff988d07e47000. 00800000:00000010:3.0:1773941498.767871:0:14663:0:(lmv_obd.c:1331:lmv_device_free()) kfreed 'lu': 64 at ffff988d02f4da00. 00800000:00000001:3.0:1773941498.767874:0:14663:0:(lmv_obd.c:1333:lmv_device_free()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941498.767877:0:14663:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d1817d800. 00000040:00000010:3.0:1773941498.767880:0:14663:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d1817d400. 00000020:00000010:3.0:1773941498.767883:0:14663:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d07233000. 00000020:00000010:3.0:1773941498.767886:0:14663:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d1817d600. 00040000:00000010:3.0:1773941498.767889:0:14663:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d1a2ec200. 00040000:00000010:3.0:1773941498.767892:0:14663:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d1817c600. 00000020:00000010:3.0:1773941498.767895:0:14663:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d1817c400. 00000020:00000001:3.0:1773941498.767898:0:14663:0:(obd_class.h:647:obd_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941498.767900:0:14663:0:(genops.c:66:obd_device_free()) slab-freed '(obd)': 6584 at ffff988d089a8000. 00000020:00000001:3.0:1773941498.767906:0:14663:0:(obd_config.c:923:class_detach()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941498.767909:0:14663:0:(obd_config.c:1655:class_process_config()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:3.0:1773941498.767912:0:14663:0:(obd_config.c:2523:class_manual_cleanup()) kfreed 'lcfg': 80 at ffff988d111f7780. 00000020:00000001:3.0:1773941498.767915:0:14663:0:(obd_config.c:2524:class_manual_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941498.767918:0:14663:0:(obd_config.c:2489:class_manual_cleanup()) Process entered 00000020:01000000:3.0:1773941498.767920:0:14663:0:(obd_config.c:2502:class_manual_cleanup()) Manual cleanup of lustre-MDT0000-mdc-ffff988d08f0a000 (flags='') 00000020:00000010:3.0:1773941498.767924:0:14663:0:(obd_config.c:2506:class_manual_cleanup()) kmalloced '(lcfg)': 88 at ffff988d111f7780. 00000020:00000080:3.0:1773941498.767927:0:14663:0:(obd_config.c:1506:class_process_config()) processing cmd: cf004 00000020:00000001:3.0:1773941498.767929:0:14663:0:(obd_config.c:936:class_cleanup()) Process entered 00000020:00000080:3.0:1773941498.767931:0:14663:0:(obd_config.c:991:class_cleanup()) lustre-MDT0000-mdc-ffff988d08f0a000: forcing exports to disconnect: 0/2 00000020:00000001:3.0:1773941498.767934:0:14663:0:(genops.c:1562:class_disconnect_exports()) Process entered 00000020:00080000:3.0:1773941498.767936:0:14663:0:(genops.c:1577:class_disconnect_exports()) OBD device 3 (ffff988d089ab370) has no exports 00000020:00000001:3.0:1773941498.767938:0:14663:0:(genops.c:1578:class_disconnect_exports()) Process leaving 00000020:00000001:3.0:1773941498.767940:0:14663:0:(obd_class.h:604:obd_precleanup()) Process entered 00000020:00000010:3.0:1773941498.767943:0:14663:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d1817c400. 00000020:00000010:3.0:1773941498.767946:0:14663:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d1817c600. 00000020:00000010:3.0:1773941498.767954:0:14663:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d111f70c0. 00008000:00000010:3.0:1773941498.767959:0:14663:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d107f0000. 00000020:00000010:3.0:1773941498.767990:0:14663:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff988d03ec7e40. 00000002:00000001:3.0:1773941498.767995:0:14663:0:(mdc_dev.c:1760:mdc_device_fini()) Process entered 00000008:00000001:3.0:1773941498.767998:0:14663:0:(osc_request.c:4123:osc_precleanup_common()) Process entered 00000008:00000001:3.0:1773941498.768004:0:14663:0:(obd_class.h:652:obd_cleanup_client_import()) Process entered 00000008:01000000:3.0:1773941498.768006:0:14663:0:(obd_class.h:663:obd_cleanup_client_import()) lustre-MDT0000-mdc-ffff988d08f0a000: client import never connected 00000100:00000001:3.0:1773941498.768009:0:14663:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941498.768010:0:14663:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-MDT0000_UUID INVALID 00000100:00000001:3.0:1773941498.768012:0:14663:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00000001:3.0:1773941498.768014:0:14663:0:(client.c:3065:ptlrpc_free_committed()) Process entered 00000100:00100000:3.0:1773941498.768016:0:14663:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff988d08f0a000: committing for last_committed 0 gen 3 00000100:00000040:3.0:1773941498.768019:0:14663:0:(client.c:3191:ptlrpc_free_committed()) lustre-MDT0000-mdc-ffff988d08f0a000: normal: skip=0 replay=0/0 committed=0/0 00000100:00000001:3.0:1773941498.768022:0:14663:0:(client.c:3192:ptlrpc_free_committed()) Process leaving 00000100:00000001:3.0:1773941498.768023:0:14663:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941498.768024:0:14663:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941498.768026:0:14663:0:(obd_class.h:1215:obd_import_event()) Process entered 00000002:00000001:3.0:1773941498.768028:0:14663:0:(mdc_request.c:2862:mdc_import_event()) Process entered 00000002:00000001:3.0:1773941498.768030:0:14663:0:(mdc_request.c:2936:mdc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941498.768032:0:14663:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00100000:3.0:1773941498.768033:0:14663:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:3.0:1773941498.768035:0:14663:0:(obd_class.h:1215:obd_import_event()) Process entered 00000002:00000001:3.0:1773941498.768035:0:14663:0:(mdc_request.c:2862:mdc_import_event()) Process entered 00000001:00000001:3.0:1773941498.768037:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941498.768038:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941498.768420:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941498.768423:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941498.768424:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941498.768740:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941498.768741:0:14663:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:3.0:1773941498.768745:0:14663:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff988d08a3fcf0. 00000020:00000010:3.0:1773941498.768747:0:14663:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d1817d600. 00000020:00000010:3.0:1773941498.768749:0:14663:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d1817d400. 00000020:00000010:3.0:1773941498.768752:0:14663:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d111f7c60. 00008000:00000010:3.0:1773941498.768755:0:14663:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d107f0e20. 00000020:00000010:3.0:1773941498.768758:0:14663:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d1817d800. 00008000:00000010:3.0:1773941498.768765:0:14663:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff988d209194e0. 00000020:00000001:3.0:1773941498.768768:0:14663:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446630330235616504 : -113743473935112 : ffff988d08a3fcf8) 00000020:00001000:3.0:1773941498.768770:0:14663:0:(cl_object.c:831:cl_env_get()) 1@ffff988d08a3fcf0 00000008:00000001:3.0:1773941498.768772:0:14663:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:3.0:1773941498.768773:0:14663:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:3.0:1773941498.768774:0:14663:0:(osc_cache.c:2316:osc_next_obj()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1773941498.768776:0:14663:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000001:00000001:3.0:1773941498.768777:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941498.768778:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941498.769132:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:3.0:1773941498.769134:0:14663:0:(cl_object.c:925:cl_env_put()) 1@ffff988d08a3fcf0 00000020:00000010:3.0:1773941498.769137:0:14663:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d1817d400. 00000020:00000010:3.0:1773941498.769140:0:14663:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d111f7c60. 00008000:00000010:3.0:1773941498.769142:0:14663:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d107f0e20. 00000020:00000010:3.0:1773941498.769144:0:14663:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d1817d600. 00008000:00000010:3.0:1773941498.769146:0:14663:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff988d209194e0. 00000020:00000010:3.0:1773941498.769148:0:14663:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d1817d800. 00000020:00000010:3.0:1773941498.769149:0:14663:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff988d08a3fcf0. 00000001:00000001:3.0:1773941498.769152:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941498.769153:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941498.769476:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941498.769478:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941498.769479:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941498.769787:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:3.0:1773941498.769789:0:14663:0:(mdc_request.c:2936:mdc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941498.769790:0:14663:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000040:3.0:1773941498.769794:0:14663:0:(genops.c:1229:class_import_get()) import ffff988d035c7000 refcount=3 obd=lustre-MDT0000-mdc-ffff988d08f0a000 00000020:00000001:3.0:1773941498.769796:0:14663:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941498.769797:0:14663:0:(genops.c:1242:class_import_put()) import ffff988d035c7000 refcount=2 obd=lustre-MDT0000-mdc-ffff988d08f0a000 00000020:00000001:3.0:1773941498.769798:0:14663:0:(genops.c:1249:class_import_put()) Process leaving 00000020:00000001:3.0:1773941498.769800:0:14663:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941498.769801:0:14663:0:(genops.c:1242:class_import_put()) import ffff988d035c7000 refcount=1 obd=lustre-MDT0000-mdc-ffff988d08f0a000 00000020:00000001:3.0:1773941498.769802:0:14663:0:(genops.c:1249:class_import_put()) Process leaving 00000008:00000001:3.0:1773941498.769803:0:14663:0:(obd_class.h:670:obd_cleanup_client_import()) Process leaving 00000008:00000001:3.0:1773941498.769805:0:14663:0:(osc_request.c:4138:osc_precleanup_common()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:3.0:1773941498.769807:0:14663:0:(mdc_changelog.c:886:mdc_changelog_cdev_finish()) Process entered 00000002:00000001:3.0:1773941498.769816:0:14663:0:(mdc_changelog.c:136:chlg_dev_clear()) Process entered 00000002:00000010:3.0:1773941498.770027:0:14663:0:(mdc_changelog.c:128:chlg_device_release()) kfreed 'entry': 1304 at ffff988d035c5800. 00000002:00000001:3.0:1773941498.770034:0:14663:0:(mdc_changelog.c:143:chlg_dev_clear()) Process leaving 00000002:00000001:3.0:1773941498.770035:0:14663:0:(mdc_changelog.c:892:mdc_changelog_cdev_finish()) Process leaving 00000002:00000001:3.0:1773941498.770037:0:14663:0:(mdc_request.c:3022:mdc_llog_finish()) Process entered 00000002:00000040:3.0:1773941498.770039:0:14663:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6e40 : new refcount 2 00000040:00000001:3.0:1773941498.770041:0:14663:0:(llog_obd.c:93:llog_cleanup()) Process entered 00000040:00000040:3.0:1773941498.770042:0:14663:0:(lustre_log.h:400:llog_ctxt_put()) PUTting ctxt ffff988d02eb6e40 : new refcount 1 00000020:00000040:3.0:1773941498.770045:0:14663:0:(genops.c:898:class_export_put()) PUTting export ffff988d035c4000 : new refcount 1 00000020:00000001:3.0:1773941498.770046:0:14663:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941498.770047:0:14663:0:(genops.c:1242:class_import_put()) import ffff988d035c7000 refcount=0 obd=lustre-MDT0000-mdc-ffff988d08f0a000 00000020:00000040:3.0:1773941498.770049:0:14663:0:(genops.c:1245:class_import_put()) final put import ffff988d035c7000 00000020:00000001:3.0:1773941498.770056:0:14663:0:(genops.c:1249:class_import_put()) Process leaving 00000040:00000010:3.0:1773941498.770057:0:14663:0:(llog_obd.c:46:llog_ctxt_destroy()) kfreed 'ctxt': 168 at ffff988d02eb6e40. 00000020:00000001:3.0:1773941498.770066:0:8:0:(genops.c:1200:obd_zombie_import_free()) Process entered 00000020:00000080:3.0:1773941498.770067:0:8:0:(genops.c:1202:obd_zombie_import_free()) destroying import ffff988d035c7000 for lustre-MDT0000-mdc-ffff988d08f0a000 00000020:00000040:3.0:1773941498.770072:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 6 to 192.168.204.114@tcp 00000020:00000040:3.0:1773941498.770074:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 5 to 192.168.204.114@tcp 00000020:00000010:3.0:1773941498.770076:0:8:0:(genops.c:1213:obd_zombie_import_free()) kfreed 'imp_conn': 88 at ffff988d18350de0. 00000020:00000040:3.0:1773941498.770078:0:8:0:(obd_config.c:1058:class_decref()) Decref lustre-MDT0000-mdc-ffff988d08f0a000 (ffff988d089ab370) now 4 - import 00000020:00000010:3.0:1773941498.770080:0:8:0:(genops.c:1220:obd_zombie_import_free()) kfreed 'imp': 1712 at ffff988d035c7000. 00000020:00000001:3.0:1773941498.770082:0:8:0:(genops.c:1221:obd_zombie_import_free()) Process leaving 00000040:00000001:3.0:1773941498.770087:0:14663:0:(llog_obd.c:122:llog_cleanup()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:3.0:1773941498.770088:0:14663:0:(mdc_request.c:3027:mdc_llog_finish()) Process leaving 00000020:00000010:3.0:1773941498.770090:0:14663:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 640 at ffff988d065fa000. 00000020:00000010:3.0:1773941498.770093:0:14663:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 640 at ffff988d1118a000. 00000020:00000010:3.0:1773941498.770095:0:14663:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 640 at ffff988d111d1400. 00000020:00000010:3.0:1773941498.770099:0:14663:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 512 at ffff988d04ce4800. 00000020:00000010:3.0:1773941498.770101:0:14663:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff988d203ff000. 00000020:00000010:3.0:1773941498.770267:0:14663:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 4608 at ffff988d10800000. 00000020:00000010:3.0:1773941498.770271:0:14663:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 4608 at ffff988d0352a000. 00000020:00000010:3.0:1773941498.770274:0:14663:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 4608 at ffff988d18b14000. 00000020:00000010:3.0:1773941498.770277:0:14663:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 4608 at ffff988d06468000. 00000020:00000010:3.0:1773941498.770281:0:14663:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 3680 at ffff988d02d6b000. 00000020:00000010:3.0:1773941498.770290:0:14663:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff988d203ff800. 00000002:00000001:3.0:1773941498.770293:0:14663:0:(mdc_dev.c:1768:mdc_device_fini()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941498.770296:0:14663:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff988d03ec7e40. 00000020:00000010:3.0:1773941498.770300:0:14663:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d1817c600. 00000020:00000010:3.0:1773941498.770302:0:14663:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d111f70c0. 00008000:00000010:3.0:1773941498.770304:0:14663:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d107f0000. 00000020:00000010:3.0:1773941498.770308:0:14663:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d1817c400. 00000020:00000001:3.0:1773941498.770311:0:14663:0:(obd_class.h:623:obd_precleanup()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941498.770325:0:14663:0:(hash.c:1113:cfs_hash_destroy()) Process entered 00000001:00000010:3.0:1773941498.770329:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff988d04ce5c00. 00000001:00000010:3.0:1773941498.770332:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff988d04ce5200. 00000001:00000010:3.0:1773941498.770334:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff988d04ce4000. 00000001:00000010:3.0:1773941498.770336:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff988d04ce5a00. 00000001:00000010:3.0:1773941498.770338:0:14663:0:(hash.c:913:cfs_hash_buckets_free()) kfreed 'buckets': 32 at ffff988d107cf7e0. 00000001:00000010:3.0:1773941498.770339:0:14663:0:(hash.c:1163:cfs_hash_destroy()) kfreed 'hs': 232 at ffff988d02cba200. 00000001:00000001:3.0:1773941498.770341:0:14663:0:(hash.c:1165:cfs_hash_destroy()) Process leaving 00000020:00000040:3.0:1773941498.770342:0:14663:0:(obd_config.c:1058:class_decref()) Decref lustre-MDT0000-mdc-ffff988d08f0a000 (ffff988d089ab370) now 3 - setup 00000020:00000001:3.0:1773941498.770344:0:14663:0:(obd_config.c:1021:class_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941498.770346:0:14663:0:(obd_config.c:1659:class_process_config()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000080:3.0:1773941498.770348:0:14663:0:(obd_config.c:1506:class_process_config()) processing cmd: cf002 00000020:00000001:3.0:1773941498.770351:0:14663:0:(obd_config.c:899:class_detach()) Process entered 00000020:00000040:3.0:1773941498.770353:0:14663:0:(obd_config.c:1058:class_decref()) Decref lustre-MDT0000-mdc-ffff988d08f0a000 (ffff988d089ab370) now 2 - obd_device_list 00000020:00000080:3.0:1773941498.770355:0:14663:0:(obd_config.c:919:class_detach()) detach on obd lustre-MDT0000-mdc-ffff988d08f0a000 (uuid e9ee4b2c-2bac-44e1-850a-8bf1479379ac) 00000020:00000040:3.0:1773941498.770357:0:14663:0:(obd_config.c:1058:class_decref()) Decref lustre-MDT0000-mdc-ffff988d08f0a000 (ffff988d089ab370) now 1 - newdev 00000020:00000040:3.0:1773941498.770359:0:14663:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d035c4000 with handle 0x8e86767622e2880 from hash 00000020:00000040:3.0:1773941498.770361:0:14663:0:(genops.c:898:class_export_put()) PUTting export ffff988d035c4000 : new refcount 0 00000020:00000080:3.0:1773941498.770362:0:14663:0:(genops.c:904:class_export_put()) final put ffff988d035c4000/e9ee4b2c-2bac-44e1-850a-8bf1479379ac 00000020:00000001:3.0:1773941498.770364:0:14663:0:(lprocfs_status_server.c:686:lprocfs_exp_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941498.770365:0:14663:0:(genops.c:857:class_export_destroy()) Process entered 00000020:00000080:3.0:1773941498.770367:0:14663:0:(genops.c:863:class_export_destroy()) destroying export ffff988d035c4000/e9ee4b2c-2bac-44e1-850a-8bf1479379ac for lustre-MDT0000-mdc-ffff988d08f0a000 00000020:00000001:3.0:1773941498.770369:0:14663:0:(obd_class.h:1017:obd_destroy_export()) Process entered 00000020:00000001:3.0:1773941498.770370:0:14663:0:(obd_class.h:1021:obd_destroy_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941498.770377:0:14663:0:(genops.c:879:class_export_destroy()) kfreed_rcu 'exp': 1208 at ffff988d035c4000. 00000020:00000001:3.0:1773941498.770382:0:14663:0:(genops.c:880:class_export_destroy()) Process leaving 00000020:00000040:3.0:1773941498.770383:0:14663:0:(genops.c:437:class_free_dev()) Release obd device lustre-MDT0000-mdc-ffff988d08f0a000 obd_type name = mdc 00000020:01000000:3.0:1773941498.770385:0:14663:0:(genops.c:440:class_free_dev()) finishing cleanup of obd lustre-MDT0000-mdc-ffff988d08f0a000 (e9ee4b2c-2bac-44e1-850a-8bf1479379ac) 00000020:00000001:3.0:1773941498.770386:0:14663:0:(obd_class.h:633:obd_cleanup()) Process entered 00000020:00000010:3.0:1773941498.770389:0:14663:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d1817c400. 00000020:00000010:3.0:1773941498.770391:0:14663:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d1817c600. 00000020:00000010:3.0:1773941498.770393:0:14663:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d111f70c0. 00008000:00000010:3.0:1773941498.770397:0:14663:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d107f0000. 00000008:00000001:3.0:1773941498.770400:0:14663:0:(osc_request.c:4146:osc_cleanup_common()) Process entered 00010000:00000001:3.0:1773941498.770405:0:14663:0:(ldlm_lib.c:631:client_obd_cleanup()) Process entered 00010000:00000001:3.0:1773941498.770407:0:14663:0:(ldlm_resource.c:1304:ldlm_namespace_free_post()) Process entered 00010000:00000001:3.0:1773941498.770410:0:14663:0:(ldlm_pool.c:931:ldlm_pool_fini()) Process entered 00000020:00000010:3.0:1773941498.770433:0:14663:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 448 at ffff988d0381e000. 00000020:00000010:3.0:1773941498.770435:0:14663:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 448 at ffff988d08a4e400. 00000020:00000010:3.0:1773941498.770437:0:14663:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 448 at ffff988d0350ec00. 00000020:00000010:3.0:1773941498.770439:0:14663:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 352 at ffff988d04ce4200. 00000020:00000010:3.0:1773941498.770442:0:14663:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff988d203fdc00. 00010000:00000001:3.0:1773941498.770470:0:14663:0:(ldlm_pool.c:934:ldlm_pool_fini()) Process leaving 00000020:00000010:3.0:1773941498.770478:0:14663:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 128 at ffff988d181fe480. 00000020:00000010:3.0:1773941498.770480:0:14663:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 128 at ffff988d0716b480. 00000020:00000010:3.0:1773941498.770482:0:14663:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 128 at ffff988d107f4f80. 00000020:00000010:3.0:1773941498.770484:0:14663:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 96 at ffff988d18302960. 00000020:00000010:3.0:1773941498.770487:0:14663:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff988d203fd400. 00000001:00000001:3.0:1773941498.770522:0:14663:0:(hash.c:1113:cfs_hash_destroy()) Process entered 00000001:00000010:3.0:1773941498.770819:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81cb1000. 00000001:00000010:3.0:1773941498.770825:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81cbb000. 00000001:00000010:3.0:1773941498.770827:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81cc5000. 00000001:00000010:3.0:1773941498.770828:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81cd9000. 00000001:00000010:3.0:1773941498.770830:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81ce3000. 00000001:00000010:3.0:1773941498.770831:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81cf9000. 00000001:00000010:3.0:1773941498.770832:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81d03000. 00000001:00000010:3.0:1773941498.770834:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81d0d000. 00000001:00000010:3.0:1773941498.770841:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81d17000. 00000001:00000010:3.0:1773941498.770843:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81d21000. 00000001:00000010:3.0:1773941498.770845:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81d2b000. 00000001:00000010:3.0:1773941498.770846:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81d35000. 00000001:00000010:3.0:1773941498.770848:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81d3f000. 00000001:00000010:3.0:1773941498.770849:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81d51000. 00000001:00000010:3.0:1773941498.770850:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81d5b000. 00000001:00000010:3.0:1773941498.770851:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81d65000. 00000001:00000010:3.0:1773941498.770852:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81d79000. 00000001:00000010:3.0:1773941498.770853:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81d91000. 00000001:00000010:3.0:1773941498.770855:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81d9b000. 00000001:00000010:3.0:1773941498.770856:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81da5000. 00000001:00000010:3.0:1773941498.770858:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81daf000. 00000001:00000010:3.0:1773941498.770860:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81db9000. 00000001:00000010:3.0:1773941498.770862:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81dc3000. 00000001:00000010:3.0:1773941498.770864:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81e01000. 00000001:00000010:3.0:1773941498.770866:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81e0b000. 00000001:00000010:3.0:1773941498.770868:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81e15000. 00000001:00000010:3.0:1773941498.770870:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81e1f000. 00000001:00000010:3.0:1773941498.770871:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81e29000. 00000001:00000010:3.0:1773941498.770873:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81e33000. 00000001:00000010:3.0:1773941498.770875:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81e3d000. 00000001:00000010:3.0:1773941498.770877:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81e47000. 00000001:00000010:3.0:1773941498.770880:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81e51000. 00000001:00000010:3.0:1773941498.770882:0:14663:0:(hash.c:913:cfs_hash_buckets_free()) kfreed 'buckets': 256 at ffff988d02cbae00. 00000001:00000010:3.0:1773941498.770885:0:14663:0:(hash.c:1163:cfs_hash_destroy()) kfreed 'hs': 280 at ffff988d04ce4e00. 00000001:00000001:3.0:1773941498.770887:0:14663:0:(hash.c:1165:cfs_hash_destroy()) Process leaving 00010000:00000010:3.0:1773941498.770890:0:14663:0:(ldlm_resource.c:1327:ldlm_namespace_free_post()) kfreed 'ns': 848 at ffff988d203fe000. 00010000:00000001:3.0:1773941498.770895:0:14663:0:(ldlm_lockd.c:3044:ldlm_put_ref()) Process entered 00010000:00000001:1.0:1773941498.771956:0:14663:0:(ldlm_lockd.c:3058:ldlm_put_ref()) Process leaving 00010000:00000001:1.0:1773941498.771994:0:14663:0:(ldlm_resource.c:1329:ldlm_namespace_free_post()) Process leaving 00010000:00000001:1.0:1773941498.771998:0:14663:0:(obd_class.h:652:obd_cleanup_client_import()) Process entered 00010000:00000001:1.0:1773941498.772001:0:14663:0:(obd_class.h:670:obd_cleanup_client_import()) Process leaving 00010000:00000001:1.0:1773941498.772003:0:14663:0:(ldlm_lockd.c:3044:ldlm_put_ref()) Process entered 00010000:00000001:1.0:1773941498.772005:0:14663:0:(ldlm_lockd.c:3058:ldlm_put_ref()) Process leaving 00010000:00000010:1.0:1773941498.772008:0:14663:0:(ldlm_lib.c:642:client_obd_cleanup()) kfreed 'cli->cl_mod_tag_bitmap': 64 at ffff988d02f4d680. 00010000:00000010:1.0:1773941498.772015:0:14663:0:(ldlm_lib.c:647:client_obd_cleanup()) kfreed 'cli->cl_read_io_latency_by_size': 3920 at ffff988d02d6d000. 00010000:00000010:1.0:1773941498.772021:0:14663:0:(ldlm_lib.c:651:client_obd_cleanup()) kfreed 'cli->cl_write_io_latency_by_size': 3920 at ffff988d02d69000. 00010000:00000001:1.0:1773941498.772026:0:14663:0:(ldlm_lib.c:654:client_obd_cleanup()) Process leaving 00000008:00000001:1.0:1773941498.772035:0:14663:0:(osc_request.c:4167:osc_cleanup_common()) Process leaving (rc=0 : 0 : 0) 00000002:00000010:1.0:1773941498.772040:0:14663:0:(mdc_dev.c:1712:mdc_device_free()) kfreed 'osc': 104 at ffff988d03580080. 00000020:00000010:1.0:1773941498.772044:0:14663:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d1817c600. 00000020:00000010:1.0:1773941498.772054:0:14663:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d111f70c0. 00008000:00000010:1.0:1773941498.772058:0:14663:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d107f0000. 00000020:00000010:1.0:1773941498.772061:0:14663:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d1817c400. 00000020:00000001:1.0:1773941498.772065:0:14663:0:(obd_class.h:647:obd_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941498.772067:0:14663:0:(genops.c:66:obd_device_free()) slab-freed '(obd)': 6584 at ffff988d089ab370. 00000020:00000001:1.0:1773941498.772081:0:14663:0:(obd_config.c:923:class_detach()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941498.772084:0:14663:0:(obd_config.c:1655:class_process_config()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1773941498.772086:0:14663:0:(obd_config.c:2523:class_manual_cleanup()) kfreed 'lcfg': 88 at ffff988d111f7780. 00000020:00000001:1.0:1773941498.772090:0:14663:0:(obd_config.c:2524:class_manual_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941498.772093:0:14663:0:(obd_config.c:2489:class_manual_cleanup()) Process entered 00000020:01000000:1.0:1773941498.772094:0:14663:0:(obd_config.c:2502:class_manual_cleanup()) Manual cleanup of lustre-MDT0001-mdc-ffff988d08f0a000 (flags='') 00000020:00000010:1.0:1773941498.772098:0:14663:0:(obd_config.c:2506:class_manual_cleanup()) kmalloced '(lcfg)': 88 at ffff988d054f3900. 00000020:00000080:1.0:1773941498.772100:0:14663:0:(obd_config.c:1506:class_process_config()) processing cmd: cf004 00000020:00000001:1.0:1773941498.772103:0:14663:0:(obd_config.c:936:class_cleanup()) Process entered 00000020:00000080:1.0:1773941498.772106:0:14663:0:(obd_config.c:991:class_cleanup()) lustre-MDT0001-mdc-ffff988d08f0a000: forcing exports to disconnect: 0/2 00000020:00000001:1.0:1773941498.772109:0:14663:0:(genops.c:1562:class_disconnect_exports()) Process entered 00000020:00080000:1.0:1773941498.772111:0:14663:0:(genops.c:1577:class_disconnect_exports()) OBD device 4 (ffff988d089acd28) has no exports 00000020:00000001:1.0:1773941498.772113:0:14663:0:(genops.c:1578:class_disconnect_exports()) Process leaving 00000020:00000001:1.0:1773941498.772115:0:14663:0:(obd_class.h:604:obd_precleanup()) Process entered 00000020:00000010:1.0:1773941498.772125:0:14663:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d08a4f800. 00000020:00000010:1.0:1773941498.772128:0:14663:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d08a4e800. 00000020:00000010:1.0:1773941498.772131:0:14663:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d054f38a0. 00008000:00000010:1.0:1773941498.772137:0:14663:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d08339530. 00000020:00000010:1.0:1773941498.772143:0:14663:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff988d04d676c0. 00000002:00000001:1.0:1773941498.772147:0:14663:0:(mdc_dev.c:1760:mdc_device_fini()) Process entered 00000008:00000001:1.0:1773941498.772149:0:14663:0:(osc_request.c:4123:osc_precleanup_common()) Process entered 00000008:00000001:1.0:1773941498.772156:0:14663:0:(obd_class.h:652:obd_cleanup_client_import()) Process entered 00000008:01000000:1.0:1773941498.772166:0:14663:0:(obd_class.h:663:obd_cleanup_client_import()) lustre-MDT0001-mdc-ffff988d08f0a000: client import never connected 00000100:00000001:1.0:1773941498.772170:0:14663:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941498.772172:0:14663:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import lustre-MDT0001_UUID INVALID 00000100:00000001:1.0:1773941498.772175:0:14663:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00000001:1.0:1773941498.772177:0:14663:0:(client.c:3065:ptlrpc_free_committed()) Process entered 00000100:00100000:1.0:1773941498.772180:0:14663:0:(client.c:3081:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff988d08f0a000: committing for last_committed 0 gen 3 00000100:00000040:1.0:1773941498.772183:0:14663:0:(client.c:3191:ptlrpc_free_committed()) lustre-MDT0001-mdc-ffff988d08f0a000: normal: skip=0 replay=0/0 committed=0/0 00000100:00000001:1.0:1773941498.772186:0:14663:0:(client.c:3192:ptlrpc_free_committed()) Process leaving 00000100:00000001:1.0:1773941498.772195:0:14663:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941498.772196:0:14663:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941498.772206:0:14663:0:(obd_class.h:1215:obd_import_event()) Process entered 00000002:00000001:1.0:1773941498.772208:0:14663:0:(mdc_request.c:2862:mdc_import_event()) Process entered 00000002:00000001:1.0:1773941498.772211:0:14663:0:(mdc_request.c:2936:mdc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.772225:0:14663:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00100000:1.0:1773941498.772228:0:14663:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:1.0:1773941498.772231:0:14663:0:(obd_class.h:1215:obd_import_event()) Process entered 00000002:00000001:1.0:1773941498.772233:0:14663:0:(mdc_request.c:2862:mdc_import_event()) Process entered 00000001:00000001:1.0:1773941498.772235:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941498.772238:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941498.772630:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941498.772632:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941498.772633:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941498.772990:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941498.772994:0:14663:0:(cl_object.c:762:cl_env_obtain()) Process entered 00000020:00000010:1.0:1773941498.772999:0:14663:0:(cl_object.c:715:cl_env_new()) slab-alloced 'cle': 144 at ffff988d189212d0. 00000020:00000010:1.0:1773941498.773004:0:14663:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d08a4e600. 00000020:00000010:1.0:1773941498.773010:0:14663:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d08a4ec00. 00000020:00000010:1.0:1773941498.773013:0:14663:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d054f34e0. 00008000:00000010:1.0:1773941498.773018:0:14663:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d0833bc08. 00000020:00000010:1.0:1773941498.773023:0:14663:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d08a4f000. 00008000:00000010:1.0:1773941498.773028:0:14663:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff988d1891f850. 00000020:00000001:1.0:1773941498.773031:0:14663:0:(cl_object.c:793:cl_env_obtain()) Process leaving (rc=18446630330502877912 : -113743206673704 : ffff988d189212d8) 00000020:00001000:1.0:1773941498.773034:0:14663:0:(cl_object.c:831:cl_env_get()) 1@ffff988d189212d0 00000008:00000001:1.0:1773941498.773038:0:14663:0:(osc_cache.c:2325:osc_check_rpcs()) Process entered 00000008:00000001:1.0:1773941498.773040:0:14663:0:(osc_cache.c:2287:osc_next_obj()) Process entered 00000008:00000001:1.0:1773941498.773043:0:14663:0:(osc_cache.c:2316:osc_next_obj()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1773941498.773046:0:14663:0:(osc_cache.c:2389:osc_check_rpcs()) Process leaving 00000001:00000001:1.0:1773941498.773047:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941498.773049:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941498.773385:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00001000:1.0:1773941498.773387:0:14663:0:(cl_object.c:925:cl_env_put()) 1@ffff988d189212d0 00000020:00000010:1.0:1773941498.773390:0:14663:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d08a4ec00. 00000020:00000010:1.0:1773941498.773392:0:14663:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d054f34e0. 00008000:00000010:1.0:1773941498.773395:0:14663:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d0833bc08. 00000020:00000010:1.0:1773941498.773398:0:14663:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d08a4e600. 00008000:00000010:1.0:1773941498.773400:0:14663:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff988d1891f850. 00000020:00000010:1.0:1773941498.773408:0:14663:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d08a4f000. 00000020:00000010:1.0:1773941498.773410:0:14663:0:(cl_object.c:750:cl_env_fini()) slab-freed '(cle)': 144 at ffff988d189212d0. 00000001:00000001:1.0:1773941498.773413:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941498.773414:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941498.773748:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941498.773751:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941498.773752:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941498.774116:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:1.0:1773941498.774119:0:14663:0:(mdc_request.c:2936:mdc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.774121:0:14663:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000040:1.0:1773941498.774126:0:14663:0:(genops.c:1229:class_import_get()) import ffff988d035c6000 refcount=3 obd=lustre-MDT0001-mdc-ffff988d08f0a000 00000020:00000001:1.0:1773941498.774129:0:14663:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941498.774132:0:14663:0:(genops.c:1242:class_import_put()) import ffff988d035c6000 refcount=2 obd=lustre-MDT0001-mdc-ffff988d08f0a000 00000020:00000001:1.0:1773941498.774134:0:14663:0:(genops.c:1249:class_import_put()) Process leaving 00000020:00000001:1.0:1773941498.774136:0:14663:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941498.774138:0:14663:0:(genops.c:1242:class_import_put()) import ffff988d035c6000 refcount=1 obd=lustre-MDT0001-mdc-ffff988d08f0a000 00000020:00000001:1.0:1773941498.774140:0:14663:0:(genops.c:1249:class_import_put()) Process leaving 00000008:00000001:1.0:1773941498.774143:0:14663:0:(obd_class.h:670:obd_cleanup_client_import()) Process leaving 00000008:00000001:1.0:1773941498.774145:0:14663:0:(osc_request.c:4138:osc_precleanup_common()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:1.0:1773941498.774148:0:14663:0:(mdc_changelog.c:886:mdc_changelog_cdev_finish()) Process entered 00000002:00000001:1.0:1773941498.774154:0:14663:0:(mdc_changelog.c:136:chlg_dev_clear()) Process entered 00000002:00000010:1.0:1773941498.774398:0:14663:0:(mdc_changelog.c:128:chlg_device_release()) kfreed 'entry': 1304 at ffff988d035c3800. 00000002:00000001:1.0:1773941498.774406:0:14663:0:(mdc_changelog.c:143:chlg_dev_clear()) Process leaving 00000002:00000001:1.0:1773941498.774408:0:14663:0:(mdc_changelog.c:892:mdc_changelog_cdev_finish()) Process leaving 00000002:00000001:1.0:1773941498.774411:0:14663:0:(mdc_request.c:3022:mdc_llog_finish()) Process entered 00000002:00000040:1.0:1773941498.774413:0:14663:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 00000040:00000001:1.0:1773941498.774416:0:14663:0:(llog_obd.c:93:llog_cleanup()) Process entered 00000040:00000040:1.0:1773941498.774419:0:14663:0:(lustre_log.h:400:llog_ctxt_put()) PUTting ctxt ffff988d02eb6540 : new refcount 1 00000020:00000040:1.0:1773941498.774423:0:14663:0:(genops.c:898:class_export_put()) PUTting export ffff988d035c7800 : new refcount 1 00000020:00000001:1.0:1773941498.774426:0:14663:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941498.774428:0:14663:0:(genops.c:1242:class_import_put()) import ffff988d035c6000 refcount=0 obd=lustre-MDT0001-mdc-ffff988d08f0a000 00000020:00000040:1.0:1773941498.774431:0:14663:0:(genops.c:1245:class_import_put()) final put import ffff988d035c6000 00000020:00000001:1.0:1773941498.774443:0:14663:0:(genops.c:1249:class_import_put()) Process leaving 00000040:00000010:1.0:1773941498.774445:0:14663:0:(llog_obd.c:46:llog_ctxt_destroy()) kfreed 'ctxt': 168 at ffff988d02eb6540. 00000020:00000001:1.0:1773941498.774461:0:8:0:(genops.c:1200:obd_zombie_import_free()) Process entered 00000020:00000080:1.0:1773941498.774464:0:8:0:(genops.c:1202:obd_zombie_import_free()) destroying import ffff988d035c6000 for lustre-MDT0001-mdc-ffff988d08f0a000 00000020:00000040:1.0:1773941498.774471:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 4 to 192.168.204.114@tcp 00000020:00000040:1.0:1773941498.774483:0:8:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000020:00000010:1.0:1773941498.774486:0:8:0:(genops.c:1213:obd_zombie_import_free()) kfreed 'imp_conn': 88 at ffff988d18302a80. 00000020:00000040:1.0:1773941498.774491:0:8:0:(obd_config.c:1058:class_decref()) Decref lustre-MDT0001-mdc-ffff988d08f0a000 (ffff988d089acd28) now 4 - import 00000020:00000010:1.0:1773941498.774495:0:8:0:(genops.c:1220:obd_zombie_import_free()) kfreed 'imp': 1712 at ffff988d035c6000. 00000020:00000001:1.0:1773941498.774500:0:8:0:(genops.c:1221:obd_zombie_import_free()) Process leaving 00000040:00000001:1.0:1773941498.774509:0:14663:0:(llog_obd.c:122:llog_cleanup()) Process leaving (rc=0 : 0 : 0) 00000002:00000001:1.0:1773941498.774512:0:14663:0:(mdc_request.c:3027:mdc_llog_finish()) Process leaving 00000020:00000010:1.0:1773941498.774516:0:14663:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 512 at ffff988d065dde00. 00000020:00000010:1.0:1773941498.774521:0:14663:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff988d203fcc00. 00000020:00000010:1.0:1773941498.775025:0:14663:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 4608 at ffff988d10802000. 00000020:00000010:1.0:1773941498.775034:0:14663:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 4608 at ffff988d18bfc000. 00000020:00000010:1.0:1773941498.775040:0:14663:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 4608 at ffff988d18b10000. 00000020:00000010:1.0:1773941498.775046:0:14663:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 4608 at ffff988d0646c000. 00000020:00000010:1.0:1773941498.775053:0:14663:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 3680 at ffff988d02ec9000. 00000020:00000010:1.0:1773941498.775059:0:14663:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff988d111d3800. 00000002:00000001:1.0:1773941498.775064:0:14663:0:(mdc_dev.c:1768:mdc_device_fini()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941498.775070:0:14663:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff988d04d676c0. 00000020:00000010:1.0:1773941498.775083:0:14663:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d08a4e800. 00000020:00000010:1.0:1773941498.775088:0:14663:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d054f38a0. 00008000:00000010:1.0:1773941498.775093:0:14663:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d08339530. 00000020:00000010:1.0:1773941498.775098:0:14663:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d08a4f800. 00000020:00000001:1.0:1773941498.775103:0:14663:0:(obd_class.h:623:obd_precleanup()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941498.775120:0:14663:0:(hash.c:1113:cfs_hash_destroy()) Process entered 00000001:00000010:1.0:1773941498.775126:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff988d04ce5e00. 00000001:00000010:1.0:1773941498.775129:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff988d04ce5800. 00000001:00000010:1.0:1773941498.775132:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff988d04ce4a00. 00000001:00000010:1.0:1773941498.775135:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff988d04ce4600. 00000001:00000010:1.0:1773941498.775139:0:14663:0:(hash.c:913:cfs_hash_buckets_free()) kfreed 'buckets': 32 at ffff988d107cfaa0. 00000001:00000010:1.0:1773941498.775143:0:14663:0:(hash.c:1163:cfs_hash_destroy()) kfreed 'hs': 232 at ffff988d21fadf00. 00000001:00000001:1.0:1773941498.775147:0:14663:0:(hash.c:1165:cfs_hash_destroy()) Process leaving 00000020:00000040:1.0:1773941498.775149:0:14663:0:(obd_config.c:1058:class_decref()) Decref lustre-MDT0001-mdc-ffff988d08f0a000 (ffff988d089acd28) now 3 - setup 00000020:00000001:1.0:1773941498.775152:0:14663:0:(obd_config.c:1021:class_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941498.775155:0:14663:0:(obd_config.c:1659:class_process_config()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000080:1.0:1773941498.775165:0:14663:0:(obd_config.c:1506:class_process_config()) processing cmd: cf002 00000020:00000001:1.0:1773941498.775170:0:14663:0:(obd_config.c:899:class_detach()) Process entered 00000020:00000040:1.0:1773941498.775176:0:14663:0:(obd_config.c:1058:class_decref()) Decref lustre-MDT0001-mdc-ffff988d08f0a000 (ffff988d089acd28) now 2 - obd_device_list 00000020:00000080:1.0:1773941498.775180:0:14663:0:(obd_config.c:919:class_detach()) detach on obd lustre-MDT0001-mdc-ffff988d08f0a000 (uuid e9ee4b2c-2bac-44e1-850a-8bf1479379ac) 00000020:00000040:1.0:1773941498.775182:0:14663:0:(obd_config.c:1058:class_decref()) Decref lustre-MDT0001-mdc-ffff988d08f0a000 (ffff988d089acd28) now 1 - newdev 00000020:00000040:1.0:1773941498.775186:0:14663:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d035c7800 with handle 0x8e86767622e2887 from hash 00000020:00000040:1.0:1773941498.775190:0:14663:0:(genops.c:898:class_export_put()) PUTting export ffff988d035c7800 : new refcount 0 00000020:00000080:1.0:1773941498.775192:0:14663:0:(genops.c:904:class_export_put()) final put ffff988d035c7800/e9ee4b2c-2bac-44e1-850a-8bf1479379ac 00000020:00000001:1.0:1773941498.775196:0:14663:0:(lprocfs_status_server.c:686:lprocfs_exp_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941498.775199:0:14663:0:(genops.c:857:class_export_destroy()) Process entered 00000020:00000080:1.0:1773941498.775201:0:14663:0:(genops.c:863:class_export_destroy()) destroying export ffff988d035c7800/e9ee4b2c-2bac-44e1-850a-8bf1479379ac for lustre-MDT0001-mdc-ffff988d08f0a000 00000020:00000001:1.0:1773941498.775205:0:14663:0:(obd_class.h:1017:obd_destroy_export()) Process entered 00000020:00000001:1.0:1773941498.775207:0:14663:0:(obd_class.h:1021:obd_destroy_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941498.775209:0:14663:0:(genops.c:879:class_export_destroy()) kfreed_rcu 'exp': 1208 at ffff988d035c7800. 00000020:00000001:1.0:1773941498.775213:0:14663:0:(genops.c:880:class_export_destroy()) Process leaving 00000020:00000040:1.0:1773941498.775233:0:14663:0:(genops.c:437:class_free_dev()) Release obd device lustre-MDT0001-mdc-ffff988d08f0a000 obd_type name = mdc 00000020:01000000:1.0:1773941498.775235:0:14663:0:(genops.c:440:class_free_dev()) finishing cleanup of obd lustre-MDT0001-mdc-ffff988d08f0a000 (e9ee4b2c-2bac-44e1-850a-8bf1479379ac) 00000020:00000001:1.0:1773941498.775238:0:14663:0:(obd_class.h:633:obd_cleanup()) Process entered 00000020:00000010:1.0:1773941498.775243:0:14663:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d08a4f800. 00000020:00000010:1.0:1773941498.775249:0:14663:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d08a4e800. 00000020:00000010:1.0:1773941498.775253:0:14663:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d054f38a0. 00008000:00000010:1.0:1773941498.775260:0:14663:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d08339530. 00000008:00000001:1.0:1773941498.775264:0:14663:0:(osc_request.c:4146:osc_cleanup_common()) Process entered 00010000:00000001:1.0:1773941498.775270:0:14663:0:(ldlm_lib.c:631:client_obd_cleanup()) Process entered 00010000:00000001:1.0:1773941498.775272:0:14663:0:(ldlm_resource.c:1304:ldlm_namespace_free_post()) Process entered 00010000:00000001:1.0:1773941498.775276:0:14663:0:(ldlm_pool.c:931:ldlm_pool_fini()) Process entered 00000020:00000010:1.0:1773941498.775327:0:14663:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 352 at ffff988d065dca00. 00000020:00000010:1.0:1773941498.775332:0:14663:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff988d203ffc00. 00010000:00000001:1.0:1773941498.775369:0:14663:0:(ldlm_pool.c:934:ldlm_pool_fini()) Process leaving 00000020:00000010:1.0:1773941498.775383:0:14663:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 96 at ffff988d20936060. 00000020:00000010:1.0:1773941498.775387:0:14663:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff988d203fd000. 00000001:00000001:1.0:1773941498.775462:0:14663:0:(hash.c:1113:cfs_hash_destroy()) Process entered 00000001:00000010:1.0:1773941498.775933:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81e5b000. 00000001:00000010:1.0:1773941498.775951:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81e65000. 00000001:00000010:1.0:1773941498.775953:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81e6f000. 00000001:00000010:1.0:1773941498.775956:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81e79000. 00000001:00000010:1.0:1773941498.775958:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81e83000. 00000001:00000010:1.0:1773941498.775984:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81e8d000. 00000001:00000010:1.0:1773941498.775987:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81e97000. 00000001:00000010:1.0:1773941498.775988:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81ea1000. 00000001:00000010:1.0:1773941498.775990:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81eb1000. 00000001:00000010:1.0:1773941498.775992:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81ec1000. 00000001:00000010:1.0:1773941498.775994:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81ed1000. 00000001:00000010:1.0:1773941498.775996:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81edb000. 00000001:00000010:1.0:1773941498.775999:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81ee5000. 00000001:00000010:1.0:1773941498.776001:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81eef000. 00000001:00000010:1.0:1773941498.776003:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81ef9000. 00000001:00000010:1.0:1773941498.776005:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81f03000. 00000001:00000010:1.0:1773941498.776007:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81f0d000. 00000001:00000010:1.0:1773941498.776010:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81f17000. 00000001:00000010:1.0:1773941498.776012:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81f21000. 00000001:00000010:1.0:1773941498.776014:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81f2b000. 00000001:00000010:1.0:1773941498.776017:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81f35000. 00000001:00000010:1.0:1773941498.776019:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81f3f000. 00000001:00000010:1.0:1773941498.776021:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81f49000. 00000001:00000010:1.0:1773941498.776023:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81f53000. 00000001:00000010:1.0:1773941498.776026:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81f5d000. 00000001:00000010:1.0:1773941498.776028:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81f67000. 00000001:00000010:1.0:1773941498.776030:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81f71000. 00000001:00000010:1.0:1773941498.776032:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81f7b000. 00000001:00000010:1.0:1773941498.776034:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81f85000. 00000001:00000010:1.0:1773941498.776036:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81f8f000. 00000001:00000010:1.0:1773941498.776039:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81f99000. 00000001:00000010:1.0:1773941498.776041:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 32944 at ffffb6ec81fa3000. 00000001:00000010:1.0:1773941498.776043:0:14663:0:(hash.c:913:cfs_hash_buckets_free()) kfreed 'buckets': 256 at ffff988d21fad100. 00000001:00000010:1.0:1773941498.776054:0:14663:0:(hash.c:1163:cfs_hash_destroy()) kfreed 'hs': 280 at ffff988d065dc800. 00000001:00000001:1.0:1773941498.776057:0:14663:0:(hash.c:1165:cfs_hash_destroy()) Process leaving 00010000:00000010:1.0:1773941498.776060:0:14663:0:(ldlm_resource.c:1327:ldlm_namespace_free_post()) kfreed 'ns': 848 at ffff988d203fd800. 00010000:00000001:1.0:1773941498.776064:0:14663:0:(ldlm_lockd.c:3044:ldlm_put_ref()) Process entered 00010000:00000001:2.0:1773941498.776990:0:14663:0:(ldlm_lockd.c:3058:ldlm_put_ref()) Process leaving 00010000:00000001:2.0:1773941498.776993:0:14663:0:(ldlm_resource.c:1329:ldlm_namespace_free_post()) Process leaving 00010000:00000001:2.0:1773941498.777005:0:14663:0:(obd_class.h:652:obd_cleanup_client_import()) Process entered 00010000:00000001:2.0:1773941498.777007:0:14663:0:(obd_class.h:670:obd_cleanup_client_import()) Process leaving 00010000:00000001:2.0:1773941498.777008:0:14663:0:(ldlm_lockd.c:3044:ldlm_put_ref()) Process entered 00010000:00000001:2.0:1773941498.777009:0:14663:0:(ldlm_lockd.c:3058:ldlm_put_ref()) Process leaving 00010000:00000010:2.0:1773941498.777011:0:14663:0:(ldlm_lib.c:642:client_obd_cleanup()) kfreed 'cli->cl_mod_tag_bitmap': 64 at ffff988d082498c0. 00010000:00000010:2.0:1773941498.777016:0:14663:0:(ldlm_lib.c:647:client_obd_cleanup()) kfreed 'cli->cl_read_io_latency_by_size': 3920 at ffff988d07e43000. 00010000:00000010:2.0:1773941498.777020:0:14663:0:(ldlm_lib.c:651:client_obd_cleanup()) kfreed 'cli->cl_write_io_latency_by_size': 3920 at ffff988d02ecf000. 00010000:00000001:2.0:1773941498.777023:0:14663:0:(ldlm_lib.c:654:client_obd_cleanup()) Process leaving 00000008:00000001:2.0:1773941498.777025:0:14663:0:(osc_request.c:4167:osc_cleanup_common()) Process leaving (rc=0 : 0 : 0) 00000002:00000010:2.0:1773941498.777028:0:14663:0:(mdc_dev.c:1712:mdc_device_free()) kfreed 'osc': 104 at ffff988d03580480. 00000020:00000010:2.0:1773941498.777032:0:14663:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d08a4e800. 00000020:00000010:2.0:1773941498.777035:0:14663:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d054f38a0. 00008000:00000010:2.0:1773941498.777039:0:14663:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d08339530. 00000020:00000010:2.0:1773941498.777042:0:14663:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d08a4f800. 00000020:00000001:2.0:1773941498.777046:0:14663:0:(obd_class.h:647:obd_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1773941498.777048:0:14663:0:(genops.c:66:obd_device_free()) slab-freed '(obd)': 6584 at ffff988d089acd28. 00000020:00000001:2.0:1773941498.777059:0:14663:0:(obd_config.c:923:class_detach()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941498.777061:0:14663:0:(obd_config.c:1655:class_process_config()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:2.0:1773941498.777063:0:14663:0:(obd_config.c:2523:class_manual_cleanup()) kfreed 'lcfg': 88 at ffff988d054f3900. 00000020:00000001:2.0:1773941498.777066:0:14663:0:(obd_config.c:2524:class_manual_cleanup()) Process leaving (rc=0 : 0 : 0) 00000080:02000400:2.0:1773941498.777069:0:14663:0:(llite_lib.c:1617:ll_put_super()) Unmounted lustre-client 00000020:00000001:2.0:1773941498.778541:0:14663:0:(obd_config.c:1231:class_del_profile()) Process entered 00000020:01000000:2.0:1773941498.778544:0:14663:0:(obd_config.c:1233:class_del_profile()) Del profile lustre-client 00000020:00000001:2.0:1773941498.778546:0:14663:0:(obd_config.c:1149:class_get_profile_nolock()) Process entered 00000020:00000001:2.0:1773941498.778548:0:14663:0:(obd_config.c:1153:class_get_profile_nolock()) Process leaving (rc=18446630330140250240 : -113743569301376 : ffff988d02f4d080) 00000020:00000010:2.0:1773941498.778552:0:14663:0:(obd_config.c:1270:class_put_profile()) kfreed 'lprof->lp_profile': 14 at ffff988d03879680. 00000020:00000010:2.0:1773941498.778560:0:14663:0:(obd_config.c:1271:class_put_profile()) kfreed 'lprof->lp_dt': 14 at ffff988d03879cd0. 00000020:00000010:2.0:1773941498.778564:0:14663:0:(obd_config.c:1272:class_put_profile()) kfreed 'lprof->lp_md': 14 at ffff988d03879540. 00000020:00000010:2.0:1773941498.778568:0:14663:0:(obd_config.c:1273:class_put_profile()) kfreed 'lprof': 48 at ffff988d02f4d080. 00000020:00000001:2.0:1773941498.778571:0:14663:0:(obd_config.c:1248:class_del_profile()) Process leaving 00000080:00000001:2.0:1773941498.778573:0:14663:0:(llite_lib.c:266:ll_free_sbi()) Process entered 00000020:00000010:2.0:1773941498.778594:0:14663:0:(cl_page.c:1415:cl_cache_decref()) kfreed 'cache': 152 at ffff988d18807840. 00000080:00000010:2.0:1773941498.778599:0:14663:0:(llite_lib.c:281:ll_free_sbi()) kfreed 'sbi->ll_foreign_symlink_prefix': 6 at ffff988d083a0eb8. 00000080:00000010:2.0:1773941498.778602:0:14663:0:(llite_lib.c:287:ll_free_sbi()) kfreed 'sbi->ll_foreign_symlink_upcall': 5 at ffff988d083a08b8. 00000080:00000010:2.0:1773941498.778636:0:14663:0:(llite_lib.c:310:ll_free_sbi()) kfreed 'sbi': 1376 at ffff988d181c3800. 00000080:00000001:2.0:1773941498.778648:0:14663:0:(llite_lib.c:312:ll_free_sbi()) Process leaving 00000020:00000001:2.0:1773941498.778650:0:14663:0:(obd_mount.c:909:lustre_common_put_super()) Process entered 00000020:01000004:2.0:1773941498.778652:0:14663:0:(obd_mount.c:911:lustre_common_put_super()) dropping sb ffff988d08f0a000 00000020:00000001:2.0:1773941498.778659:0:14663:0:(obd_mount.c:575:lustre_stop_mgc()) Process entered 00000020:00000001:2.0:1773941498.778662:0:14663:0:(obd_class.h:902:obd_disconnect()) Process entered 00010000:00000001:2.0:1773941498.778665:0:14663:0:(ldlm_lib.c:759:client_disconnect_export()) Process entered 00010000:00000040:2.0:1773941498.778667:0:14663:0:(ldlm_lib.c:772:client_disconnect_export()) disconnect MGC192.168.204.114@tcp - 1 00000100:00000001:2.0:1773941498.778671:0:14663:0:(pinger.c:388:ptlrpc_pinger_del_import()) Process entered 00000100:00080000:2.0:1773941498.778673:0:14663:0:(pinger.c:396:ptlrpc_pinger_del_import()) removing pingable import 48257395-ca75-4e5f-a6ae-43b9fcf66fe6->MGS 00000020:00000001:2.0:1773941498.778675:0:14663:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941498.778677:0:14663:0:(genops.c:1242:class_import_put()) import ffff988d035c4800 refcount=2 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941498.778679:0:14663:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941498.778681:0:14663:0:(pinger.c:401:ptlrpc_pinger_del_import()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941498.778683:0:14663:0:(ldlm_request.c:2367:ldlm_cli_cancel_unused()) Process entered 00000001:00000001:2.0:1773941498.778686:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941498.778687:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00010000:00000040:2.0:1773941498.778691:0:14663:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff988d071ee100 count: 2 00010000:00000001:2.0:1773941498.778694:0:14663:0:(ldlm_request.c:2311:ldlm_cli_cancel_unused_resource()) Process entered 00010000:00000040:2.0:1773941498.778697:0:14663:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff988d071ee100 count: 3 00010000:00000001:2.0:1773941498.778699:0:14663:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:2.0:1773941498.778705:0:14663:0:(ldlm_request.c:2161:ldlm_cancel_resource_local()) Process entered 00010000:00000001:2.0:1773941498.778710:0:14663:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:2.0:1773941498.778714:0:14663:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2864 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x1008400000000 nid: local remote: 0xd03f0241a6d949c5 expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:2.0:1773941498.778723:0:14663:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:2.0:1773941498.778726:0:14663:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2864 lrc: 2/0,0 mode: CR/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x1009400000000 nid: local remote: 0xd03f0241a6d949c5 expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1773941498.778731:0:14663:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:01000000:2.0:1773941498.778733:0:14663:0:(mgc_request.c:954:mgc_blocking_ast()) log lustre-s: disconnecting, won't requeue 10000000:00000001:2.0:1773941498.778735:0:14663:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:2.0:1773941498.778736:0:14663:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4200) refs 1 10000000:01000000:2.0:1773941498.778739:0:14663:0:(mgc_request.c:130:config_log_put()) dropping config log lustre-sptlrpc 10000000:00000001:2.0:1773941498.778741:0:14663:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941498.778742:0:14663:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941498.778753:0:14663:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941498.778754:0:14663:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941498.778756:0:14663:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941498.778758:0:14663:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941498.778759:0:14663:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941498.778760:0:14663:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941498.778761:0:14663:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941498.778763:0:14663:0:(mgc_request.c:120:config_log_put()) Process leaving 02000000:08000000:2.0:1773941498.778766:0:14663:0:(sec_config.c:500:sptlrpc_conf_free()) free sptlrpc conf lustre 02000000:00000010:2.0:1773941498.778768:0:14663:0:(sec_config.c:504:sptlrpc_conf_free()) kfreed 'conf': 120 at ffff988d107f4900. 00000020:00000040:2.0:1773941498.778772:0:14663:0:(genops.c:898:class_export_put()) PUTting export ffff988d08f0d800 : new refcount 2 10000000:00000010:2.0:1773941498.778774:0:14663:0:(mgc_request.c:144:config_log_put()) kfreed 'cld': 287 at ffff988d065c4200. 10000000:00000001:2.0:1773941498.778777:0:14663:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:2.0:1773941498.778778:0:14663:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941498.778783:0:14663:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:2.0:1773941498.778786:0:14663:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1773941498.778788:0:14663:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1773941498.778790:0:14663:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941498.778798:0:14663:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941498.778800:0:14663:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d089dfa00 with handle 0x8e86767622e2864 from hash 00010000:00000001:2.0:1773941498.778802:0:14663:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1773941498.778804:0:14663:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941498.778805:0:14663:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941498.778806:0:14663:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1773941498.778808:0:14663:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:2.0:1773941498.778810:0:14663:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=549755813888 : 549755813888 : 8000000000) 00010000:00000001:2.0:1773941498.778812:0:14663:0:(ldlm_request.c:1740:ldlm_cli_cancel_list_local()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1773941498.778814:0:14663:0:(ldlm_request.c:2211:ldlm_cancel_resource_local()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1773941498.778816:0:14663:0:(ldlm_request.c:2242:ldlm_cli_cancel_list()) Process entered 00010000:00000001:2.0:1773941498.778818:0:14663:0:(ldlm_request.c:1396:ldlm_cli_cancel_req()) Process entered 00000100:00000010:2.0:1773941498.778824:0:14663:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06504a80. 00000020:00000040:2.0:1773941498.778827:0:14663:0:(genops.c:1229:class_import_get()) import ffff988d035c4800 refcount=3 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941498.778830:0:14663:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941498.778833:0:14663:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941498.778836:0:14663:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941498.778842:0:14663:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d0350f400. 00000100:00000001:2.0:1773941498.778854:0:14663:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941498.778862:0:14663:0:(ldlm_request.c:1348:ldlm_cancel_pack()) Process entered 00010000:00010000:2.0:1773941498.778869:0:14663:0:(ldlm_request.c:1332:__ldlm_pack_lock()) ### packing ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2864 lrc: 1/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x5809400000020 nid: local remote: 0xd03f0241a6d949c5 expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941498.778874:0:14663:0:(ldlm_request.c:1378:ldlm_cancel_pack()) 1 locks packed 00010000:00000001:2.0:1773941498.778875:0:14663:0:(ldlm_request.c:1380:ldlm_cancel_pack()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941498.778878:0:14663:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:2.0:1773941498.778880:0:14663:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:2.0:1773941498.778883:0:14663:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02f7d780. 00000100:00000001:2.0:1773941498.778886:0:14663:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330140448640 : -113743569102976 : ffff988d02f7d780) 00000100:00000001:2.0:1773941498.778888:0:14663:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941498.778890:0:14663:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196576896 : -113743512974720 : ffff988d06504a80) 00000100:00000001:2.0:1773941498.778894:0:14663:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941498.778897:0:14663:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.778899:0:14663:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:2.0:1773941498.778901:0:14663:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941498.778915:0:14663:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06504a80 x1860112291476480/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941498.778922:0:14663:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941498.778924:0:14663:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941498.778928:0:14663:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06504a80 x1860112291476480/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 2 fl Rpc:QU/200/ffffffff rc -108/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941498.778933:0:14663:0:(client.c:1841:ptlrpc_send_new_req()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:2.0:1773941498.778935:0:14663:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941498.778937:0:14663:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941498.778939:0:14663:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02f7d780 going to sleep for 0 seconds 00000100:00000001:2.0:1773941498.778942:0:14663:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941498.778944:0:14663:0:(client.c:2089:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551508 : -108 : 0xffffffffffffff94) 00000100:00000001:2.0:1773941498.778946:0:14663:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941498.778948:0:14663:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941498.778950:0:14663:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941498.778954:0:14663:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06504a80 x1860112291476480/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 2 fl Interpret:QU/200/ffffffff rc -108/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941498.779002:0:14663:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06504a80 pname:cluuid:pid:xid:nid:opc:job umount:48257395-ca75-4e5f-a6ae-43b9fcf66fe6:14663:1860112291476480:192.168.204.114@tcp:103:umount.0 00000100:00000001:2.0:1773941498.779007:0:14663:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941498.779009:0:14663:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00000100:00000001:2.0:1773941498.779012:0:14663:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:2.0:1773941498.779014:0:14663:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941498.779017:0:14663:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06504a80 x1860112291476480/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 2 fl Complete:QU/200/ffffffff rc -108/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941498.779023:0:14663:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941498.779025:0:14663:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02f7d780. 00000100:00000001:2.0:1773941498.779028:0:14663:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:2.0:1773941498.779029:0:14663:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551508 : -108 : ffffffffffffff94) 00010000:00010000:2.0:1773941498.779031:0:14663:0:(ldlm_request.c:1494:ldlm_cli_cancel_req()) Got rc -108 from cancel RPC: canceling anyway 00000100:00000001:2.0:1773941498.779033:0:14663:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941498.779036:0:14663:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06504a80 x1860112291476480/t0(0) o103->MGC192.168.204.114@tcp@192.168.204.114@tcp:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl Complete:QU/200/ffffffff rc -108/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941498.779042:0:14663:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000020:00000001:2.0:1773941498.779044:0:14663:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941498.779045:0:14663:0:(genops.c:1242:class_import_put()) import ffff988d035c4800 refcount=2 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941498.779047:0:14663:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941498.779049:0:14663:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d0350f400. 02000000:00000001:2.0:1773941498.779060:0:14663:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941498.779061:0:14663:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941498.779063:0:14663:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06504a80. 00000100:00000001:2.0:1773941498.779067:0:14663:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941498.779068:0:14663:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1773941498.779070:0:14663:0:(ldlm_request.c:1501:ldlm_cli_cancel_req()) Process leaving 00010000:00010000:2.0:1773941498.779071:0:14663:0:(ldlm_request.c:2277:ldlm_cli_cancel_list()) ldlm_cli_cancel_list: -108 00010000:00000001:2.0:1773941498.779073:0:14663:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1773941498.779076:0:14663:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2864 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x5809400000020 nid: local remote: 0xd03f0241a6d949c5 expref: -99 pid: 14577 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941498.779088:0:14663:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941498.779097:0:14663:0:(ldlm_request.c:2290:ldlm_cli_cancel_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941498.779099:0:14663:0:(ldlm_request.c:2328:ldlm_cli_cancel_unused_resource()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1773941498.779102:0:14663:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d071ee100 count: 0 00000001:00000001:2.0:1773941498.779210:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941498.779213:0:14663:0:(ldlm_request.c:2379:ldlm_cli_cancel_unused()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941498.779229:0:14663:0:(ldlm_resource.c:1268:ldlm_namespace_free_prior()) Process entered 00010000:00000001:2.0:1773941498.779232:0:14663:0:(ldlm_resource.c:1207:__ldlm_namespace_free()) Process entered 00000001:00000001:2.0:1773941498.779233:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941498.779234:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941498.779236:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:2.0:1773941498.779237:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941498.779238:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941498.779240:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941498.779241:0:14663:0:(ldlm_resource.c:1251:__ldlm_namespace_free()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941498.779243:0:14663:0:(ldlm_resource.c:1293:ldlm_namespace_free_prior()) Process leaving 00000100:00000001:2.0:1773941498.779245:0:14663:0:(import.c:1924:ptlrpc_disconnect_import()) Process entered 00000100:00000001:2.0:1773941498.779248:0:14663:0:(import.c:1858:ptlrpc_disconnect_import_async()) Process entered 00000100:00000001:2.0:1773941498.779250:0:14663:0:(import.c:1750:ptlrpc_disconnect_prep_req()) Process entered 00000100:00000010:2.0:1773941498.779253:0:14663:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06504a80. 00000020:00000040:2.0:1773941498.779255:0:14663:0:(genops.c:1229:class_import_get()) import ffff988d035c4800 refcount=3 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941498.779258:0:14663:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941498.779259:0:14663:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941498.779261:0:14663:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941498.779264:0:14663:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff988d217a8a00. 00000100:00000001:2.0:1773941498.779267:0:14663:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.779269:0:14663:0:(import.c:1788:ptlrpc_disconnect_prep_req()) Process leaving (rc=18446630330196576896 : -113743512974720 : ffff988d06504a80) 00000100:00080000:2.0:1773941498.779271:0:14663:0:(import.c:65:import_set_state_nolock()) ffff988d035c4800 MGS: changing import state from FULL to CONNECTING 00000100:00000001:2.0:1773941498.779275:0:14663:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941498.779276:0:14663:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941498.779282:0:14663:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000000dade371] to pc [ptlrpcd_rcv+-1] req@ffff988d06504a80 x1860112291476608/t0(0) o251->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941498.779297:0:14663:0:(import.c:1904:ptlrpc_disconnect_import_async()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.779317:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941498.779320:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941498.779322:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941498.779329:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06504a80 x1860112291476608/t0(0) o251->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.779337:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941498.779338:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941498.779341:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941498.779343:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941498.779346:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941498.779349:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06504a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:48257395-ca75-4e5f-a6ae-43b9fcf66fe6:2400:1860112291476608:192.168.204.114@tcp:251:umount.0 00000100:00000001:1.0:1773941498.779352:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941498.779356:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941498.779358:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941498.779360:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941498.779364:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff988d08a4f000. 02000000:00000001:1.0:1773941498.779366:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941498.779372:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941498.779375:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1cd10. 00000400:00000010:1.0:1773941498.779381:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de440. 00000100:00000200:1.0:1773941498.779386:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112291476608, portal 25 00000100:00000001:1.0:1773941498.779388:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941498.779389:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196576896 : -113743512974720 : ffff988d06504a80) 00000100:00000040:1.0:1773941498.779394:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06504a80 x1860112291476608/t0(0) o251->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941508 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.779400:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941498.779403:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941498.779405:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000100:00000200:1.0:1773941498.779407:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1860112291476608, offset 0 00000400:00000200:1.0:1773941498.779413:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941498.779426:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.779432:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941498.779437:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941498.779441:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941498.779445:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941498.779448:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:1.0:1773941498.779451:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [110:110:256:4294967295] 192.168.204.114@tcp LPNI seq info [110:110:8:4294967295] 00000400:00000200:1.0:1773941498.779454:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941498.779464:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:1.0:1773941498.779469:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:1.0:1773941498.779473:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfdf00. 00000800:00000200:1.0:1773941498.779480:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:1.0:1773941498.779488:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:1.0:1773941498.779491:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfdf00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1773941498.779502:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941498.779507:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941498.779509:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294890375 00000100:00000001:1.0:1773941498.779516:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294890340 00000100:00000001:1.0:1773941498.779521:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294890375 00000100:00000001:1.0:1773941498.779526:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294890340 00000100:00000001:1.0:1773941498.779530:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941498.779532:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.779535:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.779537:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941498.779542:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941498.779544:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1773941498.779546:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941498.779548:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941498.779550:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.779551:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941498.779641:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941498.779645:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfdf00. 00000400:00000200:1.0:1773941498.779654:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941498.779660:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941498.779664:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:1.0:1773941498.779666:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:1.0:1773941498.779670:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941498.779673:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06504a80 x1860112291476608/t0(0) o251->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941508 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.779680:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941498.779683:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06504a80 x1860112291476608/t0(0) o251->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941508 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941498.779690:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.779691:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941498.781276:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941498.781290:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941498.781294:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941498.781298:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941498.781304:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941498.781311:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941498.781319:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941498.781324:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941498.781329:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941498.781333:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324603480 00000400:00000200:0.0:1773941498.781338:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941498.781344:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0x8b9 [1] + 224 00000800:00000001:0.0:1773941498.781352:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941498.781362:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941498.781365:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941498.781371:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941498.781376:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941498.781382:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06504a80 x1860112291476608/t0(0) o251->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941508 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941498.781395:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff988d06504a80 x1860112291476608/t0(0) o251->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941508 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941498.781427:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941498.781436:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941498.781440:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941498.781452:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941498.781454:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941498.781459:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941498.781461:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941498.781464:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1cd10. 00000400:00000200:0.0:1773941498.781469:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de440 00000400:00000010:0.0:1773941498.781471:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de440. 00000100:00000001:0.0:1773941498.781475:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941498.781482:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06504a80 x1860112291476608/t0(0) o251->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941508 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941498.781492:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06504a80 x1860112291476608/t0(0) o251->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941508 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941498.781500:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941498.781502:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941498.781505:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941498.781508:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941498.781511:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941498.781514:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941498.781517:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941498.781523:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d035c4c58 time=1 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941498.781527:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d035c4c10 time=1 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941498.781531:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941498.781533:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941498.781536:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:0.0:1773941498.781539:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.781542:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941498.781547:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06504a80 x1860112291476608/t0(0) o251->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941508 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941498.781555:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941498.781558:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941498.781567:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1773941498.781571:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d035c4800 MGS: changing import state from CONNECTING to CLOSED 00000100:00000001:0.0:1773941498.781575:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941498.781577:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:0.0:1773941498.781580:0:2400:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:0.0:1773941498.781582:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.781584:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941498.781586:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941498.781588:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:0.0:1773941498.781590:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.781592:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000040:0.0:1773941498.781607:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06504a80 x1860112291476608/t0(0) o251->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941508 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941498.781617:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06504a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:48257395-ca75-4e5f-a6ae-43b9fcf66fe6:2400:1860112291476608:192.168.204.114@tcp:251:umount.0 00000100:00000001:0.0:1773941498.781622:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941498.781624:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941498.781629:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06504a80 x1860112291476608/t0(0) o251->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941508 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'umount.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941498.781637:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1773941498.781640:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1773941498.781643:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff988d08a4f000. 02000000:00000001:0.0:1773941498.781647:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1773941498.781650:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941498.781652:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d035c4800 refcount=2 obd=MGC192.168.204.114@tcp 00000020:00000001:0.0:1773941498.781655:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1773941498.781658:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff988d217a8a00. 02000000:00000001:0.0:1773941498.781662:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1773941498.781664:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1773941498.781666:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06504a80. 00000100:00000001:0.0:1773941498.781670:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941498.781672:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941498.781675:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941498.781680:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941498.781682:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.781685:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941498.781693:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.781704:0:14663:0:(import.c:1967:ptlrpc_disconnect_import()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.781709:0:14663:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:0.0:1773941498.781711:0:14663:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:0.0:1773941498.781713:0:14663:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00000001:0.0:1773941498.781715:0:14663:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:0.0:1773941498.781717:0:14663:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:0.0:1773941498.781719:0:14663:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941498.781721:0:14663:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:0.0:1773941498.781723:0:14663:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.781726:0:14663:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00100000:0.0:1773941498.781728:0:14663:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:0.0:1773941498.781731:0:14663:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941498.781732:0:14663:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:0.0:1773941498.781734:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941498.781737:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941498.781740:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1773941498.781742:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941498.781744:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941498.781747:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941498.781750:0:14663:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.781752:0:14663:0:(obd_class.h:1226:obd_import_event()) Process leaving 00010000:00000001:0.0:1773941498.781756:0:14663:0:(ldlm_lib.c:818:client_disconnect_export()) Process leaving 00000020:00000001:0.0:1773941498.781759:0:14663:0:(genops.c:1462:class_disconnect()) Process entered 00000020:00000080:0.0:1773941498.781762:0:14663:0:(genops.c:1487:class_disconnect()) disconnect: cookie 0x8e86767622e285d 00000020:00000040:0.0:1773941498.781766:0:14663:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d181c6800 with handle 0x8e86767622e285d from hash 00000020:00000001:0.0:1773941498.781772:0:14663:0:(genops.c:1934:obd_stale_export_put()) Process entered 00000020:00000040:0.0:1773941498.781775:0:14663:0:(genops.c:898:class_export_put()) PUTting export ffff988d181c6800 : new refcount 1 00000020:00000001:0.0:1773941498.781777:0:14663:0:(genops.c:1959:obd_stale_export_put()) Process leaving 00000020:00000040:0.0:1773941498.781779:0:14663:0:(genops.c:898:class_export_put()) PUTting export ffff988d181c6800 : new refcount 0 00000020:00000080:0.0:1773941498.781781:0:14663:0:(genops.c:904:class_export_put()) final put ffff988d181c6800/48257395-ca75-4e5f-a6ae-43b9fcf66fe6 00000020:00000001:0.0:1773941498.781785:0:14663:0:(lprocfs_status_server.c:686:lprocfs_exp_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941498.781797:0:14663:0:(genops.c:1493:class_disconnect()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941498.781800:0:14663:0:(ldlm_lib.c:831:client_disconnect_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941498.781803:0:14663:0:(obd_class.h:914:obd_disconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941498.781806:0:14663:0:(obd_config.c:2489:class_manual_cleanup()) Process entered 00000020:01000000:0.0:1773941498.781808:0:14663:0:(obd_config.c:2502:class_manual_cleanup()) Manual cleanup of MGC192.168.204.114@tcp (flags='') 00000020:00000010:0.0:1773941498.781813:0:14663:0:(obd_config.c:2506:class_manual_cleanup()) kmalloced '(lcfg)': 72 at ffff988d06580360. 00000020:00000080:0.0:1773941498.781823:0:14663:0:(obd_config.c:1506:class_process_config()) processing cmd: cf004 00000020:00000001:0.0:1773941498.781828:0:14663:0:(obd_config.c:936:class_cleanup()) Process entered 00000020:00000080:0.0:1773941498.781831:0:14663:0:(obd_config.c:991:class_cleanup()) MGC192.168.204.114@tcp: forcing exports to disconnect: 0/3 00000020:00000001:0.0:1773941498.781835:0:14663:0:(genops.c:1562:class_disconnect_exports()) Process entered 00000020:00080000:0.0:1773941498.781838:0:14663:0:(genops.c:1577:class_disconnect_exports()) OBD device 0 (ffff988d083919b8) has no exports 00000020:00000001:0.0:1773941498.781840:0:14663:0:(genops.c:1578:class_disconnect_exports()) Process leaving 00000020:00000001:0.0:1773941498.781844:0:14663:0:(obd_class.h:604:obd_precleanup()) Process entered 00000020:00000010:0.0:1773941498.781848:0:14663:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d0381ec00. 00000020:00000010:0.0:1773941498.781854:0:14663:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d0381fe00. 00000040:00000010:0.0:1773941498.781859:0:14663:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d0381fa00. 00000020:00000010:0.0:1773941498.781866:0:14663:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d111b0000. 00000020:00000010:0.0:1773941498.781872:0:14663:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d0381e400. 00040000:00000010:0.0:1773941498.781879:0:14663:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d04c1a400. 00040000:00000010:0.0:1773941498.781884:0:14663:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d0381e000. 00000020:00000010:0.0:1773941498.781891:0:14663:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff988d108729c0. 10000000:00000001:0.0:1773941498.781895:0:14663:0:(mgc_request.c:792:mgc_device_fini()) Process entered 10000000:00000001:0.0:1773941498.781922:0:14591:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941498.781925:0:14591:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:01000000:0.0:1773941498.781933:0:14591:0:(mgc_request.c:708:mgc_requeue_thread()) Ending requeue thread 10000000:00000001:0.0:1773941498.781936:0:14591:0:(mgc_request.c:709:mgc_requeue_thread()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941498.782331:0:461:0:(genops.c:857:class_export_destroy()) Process entered 00000020:00000080:0.0:1773941498.782335:0:461:0:(genops.c:863:class_export_destroy()) destroying export ffff988d181c6800/48257395-ca75-4e5f-a6ae-43b9fcf66fe6 for MGC192.168.204.114@tcp 00000020:00000040:0.0:1773941498.782340:0:461:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000020:00000001:0.0:1773941498.782343:0:461:0:(obd_class.h:1017:obd_destroy_export()) Process entered 00000020:00000001:0.0:1773941498.782346:0:461:0:(obd_class.h:1021:obd_destroy_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1773941498.782349:0:461:0:(obd_config.c:1058:class_decref()) Decref MGC192.168.204.114@tcp (ffff988d083919b8) now 5 - export 00000020:00000010:0.0:1773941498.782353:0:461:0:(genops.c:879:class_export_destroy()) kfreed_rcu 'exp': 1208 at ffff988d181c6800. 00000020:00000001:0.0:1773941498.782360:0:461:0:(genops.c:880:class_export_destroy()) Process leaving 10000000:00000001:0.0:1773941498.782369:0:14663:0:(obd_class.h:652:obd_cleanup_client_import()) Process entered 10000000:01000000:0.0:1773941498.782372:0:14663:0:(obd_class.h:663:obd_cleanup_client_import()) MGC192.168.204.114@tcp: client import never connected 00000100:00000001:0.0:1773941498.782375:0:14663:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:0.0:1773941498.782377:0:14663:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:0.0:1773941498.782379:0:14663:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00000001:0.0:1773941498.782381:0:14663:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:0.0:1773941498.782383:0:14663:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:0.0:1773941498.782392:0:14663:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941498.782394:0:14663:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:0.0:1773941498.782397:0:14663:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.782399:0:14663:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00100000:0.0:1773941498.782401:0:14663:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:0.0:1773941498.782404:0:14663:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941498.782406:0:14663:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:0.0:1773941498.782408:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941498.782411:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941498.782415:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1773941498.782418:0:14663:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941498.782420:0:14663:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941498.782422:0:14663:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941498.782424:0:14663:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.782426:0:14663:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000040:0.0:1773941498.782431:0:14663:0:(genops.c:1229:class_import_get()) import ffff988d035c4800 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:0.0:1773941498.782434:0:14663:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941498.782436:0:14663:0:(genops.c:1242:class_import_put()) import ffff988d035c4800 refcount=2 obd=MGC192.168.204.114@tcp 00000020:00000001:0.0:1773941498.782439:0:14663:0:(genops.c:1249:class_import_put()) Process leaving 02000000:08000000:0.0:1773941498.782442:0:14663:0:(sec.c:1366:sec_cop_destroy_sec()) sec.null@ffffffffc0e99ea0: being destroyed 00000020:00000001:0.0:1773941498.782445:0:14663:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941498.782447:0:14663:0:(genops.c:1242:class_import_put()) import ffff988d035c4800 refcount=1 obd=MGC192.168.204.114@tcp 00000020:00000001:0.0:1773941498.782449:0:14663:0:(genops.c:1249:class_import_put()) Process leaving 10000000:00000001:0.0:1773941498.782451:0:14663:0:(obd_class.h:670:obd_cleanup_client_import()) Process leaving 10000000:00000001:0.0:1773941498.782453:0:14663:0:(mgc_request.c:774:mgc_llog_fini()) Process entered 10000000:00000040:0.0:1773941498.782456:0:14663:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb60c0 : new refcount 2 00000040:00000001:0.0:1773941498.782460:0:14663:0:(llog_obd.c:93:llog_cleanup()) Process entered 00000040:00000040:0.0:1773941498.782462:0:14663:0:(lustre_log.h:400:llog_ctxt_put()) PUTting ctxt ffff988d02eb60c0 : new refcount 1 00000020:00000040:0.0:1773941498.782466:0:14663:0:(genops.c:898:class_export_put()) PUTting export ffff988d08f0d800 : new refcount 1 00000020:00000001:0.0:1773941498.782469:0:14663:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941498.782471:0:14663:0:(genops.c:1242:class_import_put()) import ffff988d035c4800 refcount=0 obd=MGC192.168.204.114@tcp 00000020:00000040:0.0:1773941498.782473:0:14663:0:(genops.c:1245:class_import_put()) final put import ffff988d035c4800 00000020:00000001:0.0:1773941498.782483:0:14663:0:(genops.c:1249:class_import_put()) Process leaving 00000040:00000010:0.0:1773941498.782486:0:14663:0:(llog_obd.c:46:llog_ctxt_destroy()) kfreed 'ctxt': 168 at ffff988d02eb60c0. 00000040:00000001:0.0:1773941498.782495:0:14663:0:(llog_obd.c:122:llog_cleanup()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941498.782498:0:14663:0:(mgc_request.c:780:mgc_llog_fini()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941498.782501:0:14663:0:(mgc_request.c:816:mgc_device_fini()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1773941498.782506:0:14663:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff988d108729c0. 00000020:00000010:0.0:1773941498.782519:0:14663:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d0381fe00. 00000040:00000010:0.0:1773941498.782525:0:14663:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d0381fa00. 00000020:00000010:0.0:1773941498.782531:0:14663:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d111b0000. 00000020:00000010:0.0:1773941498.782536:0:14663:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d0381e400. 00040000:00000010:0.0:1773941498.782542:0:14663:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d04c1a400. 00040000:00000010:0.0:1773941498.782547:0:14663:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d0381e000. 00000020:00000010:0.0:1773941498.782552:0:14663:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0381ec00. 00000020:00000001:0.0:1773941498.782556:0:14663:0:(obd_class.h:623:obd_precleanup()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1773941498.782577:0:14663:0:(hash.c:1113:cfs_hash_destroy()) Process entered 00000001:00000010:0.0:1773941498.782583:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff988d065c4a00. 00000001:00000010:0.0:1773941498.782588:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff988d065c4600. 00000001:00000010:0.0:1773941498.782592:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff988d065c5a00. 00000001:00000010:0.0:1773941498.782596:0:14663:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff988d065c4800. 00000001:00000010:0.0:1773941498.782599:0:14663:0:(hash.c:913:cfs_hash_buckets_free()) kfreed 'buckets': 32 at ffff988d0300eca0. 00000001:00000010:0.0:1773941498.782604:0:14663:0:(hash.c:1163:cfs_hash_destroy()) kfreed 'hs': 232 at ffff988d02cfd100. 00000001:00000001:0.0:1773941498.782607:0:14663:0:(hash.c:1165:cfs_hash_destroy()) Process leaving 00000020:00000040:0.0:1773941498.782609:0:14663:0:(obd_config.c:1058:class_decref()) Decref MGC192.168.204.114@tcp (ffff988d083919b8) now 4 - setup 00000020:00000001:0.0:1773941498.782613:0:14663:0:(obd_config.c:1021:class_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941498.782616:0:14663:0:(obd_config.c:1659:class_process_config()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000080:0.0:1773941498.782619:0:14663:0:(obd_config.c:1506:class_process_config()) processing cmd: cf002 00000020:00000001:0.0:1773941498.782623:0:14663:0:(obd_config.c:899:class_detach()) Process entered 00000020:00000040:0.0:1773941498.782626:0:14663:0:(obd_config.c:1058:class_decref()) Decref MGC192.168.204.114@tcp (ffff988d083919b8) now 3 - obd_device_list 00000020:00000080:0.0:1773941498.782630:0:14663:0:(obd_config.c:919:class_detach()) detach on obd MGC192.168.204.114@tcp (uuid 48257395-ca75-4e5f-a6ae-43b9fcf66fe6) 00000020:00000040:0.0:1773941498.782632:0:14663:0:(obd_config.c:1058:class_decref()) Decref MGC192.168.204.114@tcp (ffff988d083919b8) now 2 - newdev 00000020:00000001:0.0:1773941498.782635:0:14663:0:(obd_config.c:923:class_detach()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941498.782638:0:14663:0:(obd_config.c:1655:class_process_config()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1773941498.782641:0:14663:0:(obd_config.c:2523:class_manual_cleanup()) kfreed 'lcfg': 72 at ffff988d06580360. 00000020:00000001:0.0:1773941498.782645:0:14663:0:(obd_config.c:2524:class_manual_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941498.782648:0:14663:0:(obd_mount.c:620:lustre_stop_mgc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941498.782651:0:14663:0:(obd_mount.c:720:lustre_put_lsi()) Process entered 00000020:01000004:0.0:1773941498.782653:0:14663:0:(obd_mount.c:724:lustre_put_lsi()) put ffff988d08f0a000 1 00000020:00000001:0.0:1773941498.782656:0:14663:0:(obd_mount.c:679:lustre_free_lsi()) Process entered 00000020:01000004:0.0:1773941498.782657:0:14663:0:(obd_mount.c:682:lustre_free_lsi()) Freeing lsi ffff988d08f0a800 00000020:00000010:0.0:1773941498.782662:0:14663:0:(obd_mount.c:631:lustre_put_lsm_free()) kfreed 'lmd->lmd_dev': 28 at ffff988d0652fec0. 00000020:00000010:0.0:1773941498.782666:0:14663:0:(obd_mount.c:632:lustre_put_lsm_free()) kfreed 'lmd->lmd_profile': 14 at ffff988d02eebe30. 00000020:00000010:0.0:1773941498.782676:0:14663:0:(obd_mount.c:635:lustre_put_lsm_free()) kfreed 'lmd->lmd_opts': 17 at ffff988d0652f600. 00000020:00000010:0.0:1773941498.782679:0:14663:0:(obd_mount.c:642:lustre_put_lsm_free()) kfreed 'lmd->lmd_params': 4096 at ffff988d035d4000. 00000020:00000010:0.0:1773941498.782684:0:14663:0:(obd_mount.c:644:lustre_put_lsm_free()) kfreed 'lmd': 128 at ffff988d0716b880. 00000020:00000010:0.0:1773941498.782688:0:14663:0:(obd_mount.c:691:lustre_free_lsi()) kfreed 'lsi': 1880 at ffff988d08f0a800. 00000020:00000001:0.0:1773941498.782693:0:14663:0:(obd_mount.c:693:lustre_free_lsi()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941498.782696:0:14663:0:(obd_mount.c:727:lustre_put_lsi()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1773941498.782700:0:14663:0:(obd_mount.c:929:lustre_common_put_super()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941498.782703:0:14663:0:(cl_object.c:885:cl_env_cache_purge()) Process entered 00000020:00000001:0.0:1773941498.782708:0:14663:0:(cl_object.c:903:cl_env_cache_purge()) Process leaving (rc=4294967295 : 4294967295 : ffffffff) 00000080:00000001:0.0:1773941498.782711:0:14663:0:(llite_lib.c:1639:ll_put_super()) Process leaving 00000020:00000001:0.0:1773941498.782785:0:461:0:(genops.c:1200:obd_zombie_import_free()) Process entered 00000020:00000080:0.0:1773941498.782788:0:461:0:(genops.c:1202:obd_zombie_import_free()) destroying import ffff988d035c4800 for MGC192.168.204.114@tcp 00000020:00000040:0.0:1773941498.782793:0:461:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 1 to 192.168.204.114@tcp 00000020:00000040:0.0:1773941498.782797:0:461:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 0 to 192.168.204.114@tcp 00000020:00000010:0.0:1773941498.782800:0:461:0:(genops.c:1213:obd_zombie_import_free()) kfreed 'imp_conn': 88 at ffff988d18350240. 00000020:00000040:0.0:1773941498.782804:0:461:0:(obd_config.c:1058:class_decref()) Decref MGC192.168.204.114@tcp (ffff988d083919b8) now 1 - import 00000020:00000040:0.0:1773941498.782808:0:461:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d08f0d800 with handle 0x8e86767622e2856 from hash 00000020:00000040:0.0:1773941498.782812:0:461:0:(genops.c:898:class_export_put()) PUTting export ffff988d08f0d800 : new refcount 0 00000020:00000080:0.0:1773941498.782814:0:461:0:(genops.c:904:class_export_put()) final put ffff988d08f0d800/48257395-ca75-4e5f-a6ae-43b9fcf66fe6 00000020:00000001:0.0:1773941498.782817:0:461:0:(lprocfs_status_server.c:686:lprocfs_exp_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941498.782820:0:461:0:(genops.c:857:class_export_destroy()) Process entered 00000020:00000080:0.0:1773941498.782822:0:461:0:(genops.c:863:class_export_destroy()) destroying export ffff988d08f0d800/48257395-ca75-4e5f-a6ae-43b9fcf66fe6 for MGC192.168.204.114@tcp 00000020:00000001:0.0:1773941498.782824:0:461:0:(obd_class.h:1017:obd_destroy_export()) Process entered 00000020:00000001:0.0:1773941498.782826:0:461:0:(obd_class.h:1021:obd_destroy_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1773941498.782829:0:461:0:(genops.c:879:class_export_destroy()) kfreed_rcu 'exp': 1208 at ffff988d08f0d800. 00000020:00000001:0.0:1773941498.782834:0:461:0:(genops.c:880:class_export_destroy()) Process leaving 00000020:00000040:0.0:1773941498.782836:0:461:0:(genops.c:437:class_free_dev()) Release obd device MGC192.168.204.114@tcp obd_type name = mgc 00000020:01000000:0.0:1773941498.782838:0:461:0:(genops.c:440:class_free_dev()) finishing cleanup of obd MGC192.168.204.114@tcp (48257395-ca75-4e5f-a6ae-43b9fcf66fe6) 00000020:00000001:0.0:1773941498.782841:0:461:0:(obd_class.h:633:obd_cleanup()) Process entered 00000020:00000010:0.0:1773941498.782845:0:461:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d0381ec00. 00000020:00000010:0.0:1773941498.782849:0:461:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d0381e000. 00000040:00000010:0.0:1773941498.782854:0:461:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d0381e400. 00000020:00000010:0.0:1773941498.782859:0:461:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d111b0000. 00000020:00000010:0.0:1773941498.782871:0:461:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d0381fa00. 00040000:00000010:0.0:1773941498.782875:0:461:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d04c1a400. 00040000:00000010:0.0:1773941498.782880:0:461:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d0381fe00. 10000000:00000001:0.0:1773941498.782884:0:461:0:(mgc_request.c:824:mgc_device_free()) Process entered 00000020:00000001:0.0:1773941498.782886:0:461:0:(obd_config.c:1281:class_del_profiles()) Process entered 00000020:00000001:0.0:1773941498.782887:0:461:0:(obd_config.c:1294:class_del_profiles()) Process leaving 00010000:00000001:0.0:1773941498.783086:0:461:0:(ldlm_lib.c:631:client_obd_cleanup()) Process entered 00010000:00000001:0.0:1773941498.783088:0:461:0:(ldlm_resource.c:1304:ldlm_namespace_free_post()) Process entered 00010000:00000001:0.0:1773941498.783092:0:461:0:(ldlm_pool.c:931:ldlm_pool_fini()) Process entered 00000020:00000010:0.0:1773941498.783131:0:461:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 448 at ffff988d0335c200. 00000020:00000010:0.0:1773941498.783137:0:461:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 352 at ffff988d0335c000. 00000020:00000010:0.0:1773941498.783142:0:461:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff988d054e1800. 00010000:00000001:0.0:1773941498.783201:0:461:0:(ldlm_pool.c:934:ldlm_pool_fini()) Process leaving 00000020:00000010:0.0:1773941498.783239:0:461:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 128 at ffff988d0716be80. 00000020:00000010:0.0:1773941498.783244:0:461:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 128 at ffff988d107f4200. 00000020:00000010:0.0:1773941498.783249:0:461:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 128 at ffff988d03580b00. 00000020:00000010:0.0:1773941498.783254:0:461:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 96 at ffff988d18350f00. 00000020:00000010:0.0:1773941498.783257:0:461:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff988d054e0c00. 00000001:00000001:0.0:1773941498.783386:0:461:0:(hash.c:1113:cfs_hash_destroy()) Process entered 00000001:00000010:0.0:1773941498.783390:0:461:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 304 at ffff988d0335d600. 00000001:00000010:0.0:1773941498.783395:0:461:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 304 at ffff988d0335de00. 00000001:00000010:0.0:1773941498.783400:0:461:0:(hash.c:913:cfs_hash_buckets_free()) kfreed 'buckets': 16 at ffff988d0633c650. 00000001:00000010:0.0:1773941498.783406:0:461:0:(hash.c:1163:cfs_hash_destroy()) kfreed 'hs': 280 at ffff988d0335d200. 00000001:00000001:0.0:1773941498.783412:0:461:0:(hash.c:1165:cfs_hash_destroy()) Process leaving 00010000:00000010:0.0:1773941498.783417:0:461:0:(ldlm_resource.c:1327:ldlm_namespace_free_post()) kfreed 'ns': 848 at ffff988d054e0400. 00010000:00000001:0.0:1773941498.783423:0:461:0:(ldlm_lockd.c:3044:ldlm_put_ref()) Process entered 00010000:00000001:0.0:1773941498.783427:0:461:0:(ldlm_lockd.c:3058:ldlm_put_ref()) Process leaving 00010000:00000001:0.0:1773941498.783431:0:461:0:(ldlm_resource.c:1329:ldlm_namespace_free_post()) Process leaving 00010000:00000001:0.0:1773941498.783435:0:461:0:(obd_class.h:652:obd_cleanup_client_import()) Process entered 00010000:00000001:0.0:1773941498.783439:0:461:0:(obd_class.h:670:obd_cleanup_client_import()) Process leaving 00010000:00000001:0.0:1773941498.783442:0:461:0:(ldlm_lockd.c:3044:ldlm_put_ref()) Process entered 00010000:00000001:0.0:1773941498.783446:0:461:0:(ldlm_lockd.c:3527:ldlm_cleanup()) Process entered 00000001:00000010:0.0:1773941498.783466:0:461:0:(shrinker_debug.c:208:ll_shrinker_free()) kfreed 's': 88 at ffff988d18350060. 00000001:00000010:0.0:1773941498.783472:0:461:0:(shrinker_debug.c:208:ll_shrinker_free()) kfreed 's': 88 at ffff988d183502a0. 00000020:00000001:3.0:1773941498.783511:0:14589:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941498.783515:0:14589:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941498.783519:0:14589:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (-1 blwis in pool) 00010000:00000001:3.0:1773941498.783523:0:14589:0:(ldlm_lockd.c:2874:ldlm_bl_thread_blwi()) Process entered 00010000:00000001:3.0:1773941498.783531:0:14589:0:(ldlm_lockd.c:2878:ldlm_bl_thread_blwi()) Process leaving (rc=2 : 2 : 2) 00000020:00000010:3.0:1773941498.783544:0:14589:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff988d03e7cf80. 00000020:00000010:3.0:1773941498.783554:0:14589:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d0335fc00. 00000020:00000010:3.0:1773941498.783559:0:14589:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d08992800. 00000020:00000010:3.0:1773941498.783565:0:14589:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff988d181f3758. 00000020:00000010:3.0:1773941498.783569:0:14589:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d0335fe00. 80000000:00000010:3.0:1773941498.783574:0:14589:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d054f3120. 40000000:00000010:3.0:1773941498.783578:0:14589:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff988d03e7c3c0. 00100000:00000010:3.0:1773941498.783583:0:14589:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff988d08993800. 00040000:00000010:3.0:1773941498.783587:0:14589:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d02cfd700. 00040000:00000010:3.0:1773941498.783590:0:14589:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d0335fa00. 00002000:00000010:3.0:1773941498.783595:0:14589:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff988d11188000. 00000020:00000010:3.0:1773941498.783599:0:14589:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0335ea00. 00010000:00000010:3.0:1773941498.783603:0:14589:0:(ldlm_lockd.c:3018:ldlm_bl_thread_main()) kfreed 'env': 56 at ffff988d03e7c640. 00010000:00000001:3.0:1773941498.783606:0:14589:0:(ldlm_lockd.c:3019:ldlm_bl_thread_main()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941498.783676:0:14588:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941498.783679:0:14588:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941498.783683:0:14588:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (-2 blwis in pool) 00010000:00000001:3.0:1773941498.783686:0:14588:0:(ldlm_lockd.c:2874:ldlm_bl_thread_blwi()) Process entered 00010000:00000001:3.0:1773941498.783688:0:14588:0:(ldlm_lockd.c:2878:ldlm_bl_thread_blwi()) Process leaving (rc=2 : 2 : 2) 00000020:00000010:3.0:1773941498.783698:0:14588:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff988d03e7c740. 00000020:00000010:3.0:1773941498.783708:0:14588:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d1961ea00. 00000020:00000010:3.0:1773941498.783714:0:14588:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d08991000. 00000020:00000010:3.0:1773941498.783719:0:14588:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff988d181f4168. 00000020:00000010:3.0:1773941498.783726:0:14588:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d1961fa00. 80000000:00000010:3.0:1773941498.783730:0:14588:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d054f3ea0. 40000000:00000010:3.0:1773941498.783734:0:14588:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff988d03e7cd40. 00100000:00000010:3.0:1773941498.783738:0:14588:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff988d08991800. 00040000:00000010:3.0:1773941498.783742:0:14588:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d02cfdc00. 00040000:00000010:3.0:1773941498.783746:0:14588:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d0335e800. 00002000:00000010:3.0:1773941498.783750:0:14588:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff988d1118a800. 00000020:00000010:3.0:1773941498.783754:0:14588:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d1961e200. 00010000:00000010:3.0:1773941498.783756:0:14588:0:(ldlm_lockd.c:3018:ldlm_bl_thread_main()) kfreed 'env': 56 at ffff988d03e7c940. 00010000:00000001:3.0:1773941498.783759:0:14588:0:(ldlm_lockd.c:3019:ldlm_bl_thread_main()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941498.783830:0:14667:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941498.783833:0:14667:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941498.783836:0:14667:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (-3 blwis in pool) 00010000:00000001:2.0:1773941498.783844:0:14667:0:(ldlm_lockd.c:2874:ldlm_bl_thread_blwi()) Process entered 00010000:00000001:2.0:1773941498.783846:0:14667:0:(ldlm_lockd.c:2878:ldlm_bl_thread_blwi()) Process leaving (rc=2 : 2 : 2) 00000020:00000010:2.0:1773941498.783858:0:14667:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff988d05437340. 00000020:00000010:2.0:1773941498.783872:0:14667:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d0350e800. 00000020:00000010:2.0:1773941498.783876:0:14667:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d05449000. 00000020:00000010:2.0:1773941498.783880:0:14667:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff988d05400000. 00000020:00000010:2.0:1773941498.783884:0:14667:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d0350fc00. 80000000:00000010:2.0:1773941498.783888:0:14667:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d08bbbd20. 40000000:00000010:2.0:1773941498.783892:0:14667:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff988d054375c0. 00100000:00000010:2.0:1773941498.783896:0:14667:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff988d05449800. 00040000:00000010:2.0:1773941498.783899:0:14667:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d217a8d00. 00040000:00000010:2.0:1773941498.783903:0:14667:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d0350f200. 00002000:00000010:2.0:1773941498.783906:0:14667:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff988d035a4400. 00000020:00000010:2.0:1773941498.783910:0:14667:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0350ea00. 00010000:00000010:2.0:1773941498.783913:0:14667:0:(ldlm_lockd.c:3018:ldlm_bl_thread_main()) kfreed 'env': 56 at ffff988d05437640. 00010000:00000001:2.0:1773941498.783917:0:14667:0:(ldlm_lockd.c:3019:ldlm_bl_thread_main()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:2.1:1773941498.784081:0:461:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d089dfa00. 00010000:00000010:2.1:1773941498.784089:0:461:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d071ee100. 00010000:00000010:2.0:1773941498.784187:0:461:0:(ldlm_lockd.c:3556:ldlm_cleanup()) kfreed 'blp': 168 at ffff988d02eb6780. 00000100:00000001:2.0:1773941498.784192:0:461:0:(service.c:3841:ptlrpc_unregister_service()) Process entered 00000100:00000200:2.0:1773941498.784194:0:461:0:(service.c:3843:ptlrpc_unregister_service()) ldlm_cbd: tearing down 00000100:00000001:2.0:1773941498.784199:0:461:0:(service.c:3384:ptlrpc_stop_all_threads()) Process entered 00000100:00000001:2.0:1773941498.784201:0:461:0:(service.c:3338:ptlrpc_svcpt_stop_threads()) Process entered 00000100:00000040:2.0:1773941498.784203:0:461:0:(service.c:3341:ptlrpc_svcpt_stop_threads()) Stopping threads for service ldlm_cbd 00000100:00000040:2.0:1773941498.784205:0:461:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cb #1 00000100:00000040:2.0:1773941498.784207:0:461:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cb #0 00000100:00000040:2.0:1773941498.784228:0:461:0:(service.c:3360:ptlrpc_svcpt_stop_threads()) waiting for stopping-thread ldlm_cb #1 00000020:00000010:1.0:1773941498.788000:0:14578:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d20904c00. 00000040:00000010:1.0:1773941498.788006:0:14578:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d20905c00. 00000020:00000010:1.0:1773941498.788010:0:14578:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d08f0d000. 00000020:00000010:1.0:1773941498.788013:0:14578:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff988d181f69a8. 00000020:00000010:1.0:1773941498.788017:0:14578:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d20905400. 80000000:00000010:1.0:1773941498.788021:0:14578:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d20938420. 40000000:00000010:1.0:1773941498.788024:0:14578:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff988d209c1080. 00100000:00000010:1.0:1773941498.788027:0:14578:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff988d08f0c000. 00040000:00000010:1.0:1773941498.788030:0:14578:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d02cfdd00. 00040000:00000010:1.0:1773941498.788033:0:14578:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff988d1a075800. 00040000:00000010:1.0:1773941498.788037:0:14578:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d20904a00. 00000004:00000010:1.0:1773941498.788039:0:14578:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff988d1977c000. 00000004:00000010:1.0:1773941498.788043:0:14578:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff988d035d7000. 00000004:00000010:1.0:1773941498.788048:0:14578:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff988d1977b000. 00002000:00000010:1.0:1773941498.788050:0:14578:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff988d1081c400. 00000004:00000010:1.0:1773941498.788053:0:14578:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff988d1081cc00. 00000020:00000010:1.0:1773941498.788055:0:14578:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0a135a00. 00000020:00000010:1.0:1773941498.788059:0:14578:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff988d209c1600. 00000100:00000010:1.0:1773941498.788065:0:14578:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff988d209c10c0. 00000100:00100000:1.0:1773941498.788067:0:14578:0:(service.c:3162:ptlrpc_main()) ldlm_cb00_000: service thread [ffff988d07ebbc00:14578] 0 exiting: rc = 1 00000020:00000010:1.0:1773941498.788106:0:14579:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d1a239e00. 00000040:00000010:1.0:1773941498.788116:0:14579:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d19734c00. 00000020:00000010:1.0:1773941498.788119:0:14579:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d1977b800. 00000020:00000010:1.0:1773941498.788121:0:14579:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff988d181f0f18. 00000020:00000010:1.0:1773941498.788125:0:14579:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d19734800. 80000000:00000010:1.0:1773941498.788127:0:14579:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d209383c0. 40000000:00000010:1.0:1773941498.788130:0:14579:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff988d209c1880. 00100000:00000010:1.0:1773941498.788133:0:14579:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff988d1977e000. 00040000:00000010:1.0:1773941498.788137:0:14579:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d02cfda00. 00040000:00000010:1.0:1773941498.788140:0:14579:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff988d19779800. 00040000:00000010:1.0:1773941498.788144:0:14579:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d19735200. 00000004:00000010:1.0:1773941498.788147:0:14579:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff988d1977f000. 00000004:00000010:1.0:1773941498.788151:0:14579:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff988d0834d000. 00000004:00000010:1.0:1773941498.788154:0:14579:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff988d1977f800. 00002000:00000010:1.0:1773941498.788157:0:14579:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff988d1081e800. 00000004:00000010:1.0:1773941498.788160:0:14579:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff988d18159c00. 00000020:00000010:1.0:1773941498.788164:0:14579:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d1a238400. 00000020:00000010:1.0:1773941498.788169:0:14579:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff988d209c1200. 00000100:00000010:1.0:1773941498.788173:0:14579:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff988d209c13c0. 00000100:00100000:1.0:1773941498.788176:0:14579:0:(service.c:3162:ptlrpc_main()) ldlm_cb00_001: service thread [ffff988d05493400:14579] 1 exiting: rc = 0 00000100:00000010:2.0:1773941498.789807:0:461:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff988d07ebbc00. 00000100:00000010:2.0:1773941498.789814:0:461:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff988d05493400. 00000100:00000001:2.0:1773941498.789818:0:461:0:(service.c:3375:ptlrpc_svcpt_stop_threads()) Process leaving 00000100:00000001:2.0:1773941498.789820:0:461:0:(service.c:3338:ptlrpc_svcpt_stop_threads()) Process entered 00000100:00000040:2.0:1773941498.789821:0:461:0:(service.c:3341:ptlrpc_svcpt_stop_threads()) Stopping threads for service ldlm_cbd 00000100:00000040:2.0:1773941498.789823:0:461:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cb #1 00000100:00000040:2.0:1773941498.789825:0:461:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cb #0 00000100:00000040:2.0:1773941498.789839:0:461:0:(service.c:3360:ptlrpc_svcpt_stop_threads()) waiting for stopping-thread ldlm_cb #1 00000020:00000010:3.0:1773941498.789852:0:14581:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d063a8000. 00000020:00000010:2.0:1773941498.789852:0:14580:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d10400a00. 00000040:00000010:2.0:1773941498.789856:0:14580:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d10401400. 00000040:00000010:3.0:1773941498.789865:0:14581:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d063a9c00. 00000020:00000010:2.0:1773941498.789867:0:14580:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d09303800. 00000020:00000010:3.0:1773941498.789870:0:14581:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d0835f000. 00000020:00000010:2.0:1773941498.789873:0:14580:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff988d05400a10. 00000020:00000010:3.0:1773941498.789876:0:14581:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff988d033469a8. 00000020:00000010:2.0:1773941498.789878:0:14580:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d10400200. 00000020:00000010:3.0:1773941498.789881:0:14581:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d063a9000. 80000000:00000010:2.0:1773941498.789883:0:14580:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d08bbb960. 80000000:00000010:3.0:1773941498.789886:0:14581:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d18350660. 40000000:00000010:2.0:1773941498.789888:0:14580:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff988d05437dc0. 40000000:00000010:3.0:1773941498.789891:0:14581:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff988d02f4d880. 00100000:00000010:2.0:1773941498.789893:0:14580:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff988d09307000. 00100000:00000010:3.0:1773941498.789896:0:14581:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff988d0835c000. 00040000:00000010:2.0:1773941498.789899:0:14580:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d0a0dbd00. 00040000:00000010:3.0:1773941498.789901:0:14581:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d02cba100. 00040000:00000010:2.0:1773941498.789904:0:14580:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff988d09306000. 00040000:00000010:3.0:1773941498.789906:0:14581:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff988d08359000. 00040000:00000010:2.0:1773941498.789909:0:14580:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d10401e00. 00040000:00000010:3.0:1773941498.789911:0:14581:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d063a9400. 00000004:00000010:2.0:1773941498.789914:0:14580:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff988d09305000. 00000004:00000010:3.0:1773941498.789916:0:14581:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff988d0835a000. 00000004:00000010:2.0:1773941498.789919:0:14580:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff988d08afb000. 00000004:00000010:3.0:1773941498.789921:0:14581:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff988d02d6c000. 00000004:00000010:2.0:1773941498.789924:0:14580:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff988d09301000. 00000004:00000010:3.0:1773941498.789926:0:14581:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff988d0835b800. 00002000:00000010:2.0:1773941498.789929:0:14580:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff988d035a4000. 00002000:00000010:3.0:1773941498.789932:0:14581:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff988d054e3400. 00000004:00000010:2.0:1773941498.789934:0:14580:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff988d035a4800. 00000004:00000010:3.0:1773941498.789938:0:14581:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff988d054e2400. 00000020:00000010:2.0:1773941498.789939:0:14580:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d10400400. 00000020:00000010:3.0:1773941498.789943:0:14581:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d05493000. 00000020:00000010:2.0:1773941498.789946:0:14580:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff988d054370c0. 00000020:00000010:3.0:1773941498.789949:0:14581:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff988d02f4d8c0. 00000100:00000010:2.0:1773941498.789952:0:14580:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff988d05437c80. 00000100:00100000:2.0:1773941498.789956:0:14580:0:(service.c:3162:ptlrpc_main()) ldlm_cb01_000: service thread [ffff988d05492000:14580] 0 exiting: rc = 1 00000100:00000010:3.0:1773941498.789957:0:14581:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff988d02f4dac0. 00000100:00100000:3.0:1773941498.789983:0:14581:0:(service.c:3162:ptlrpc_main()) ldlm_cb01_001: service thread [ffff988d05493c00:14581] 1 exiting: rc = 0 00000100:00000010:2.0:1773941498.790081:0:461:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff988d05492000. 00000100:00000010:2.0:1773941498.790084:0:461:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff988d05493c00. 00000100:00000001:2.0:1773941498.790087:0:461:0:(service.c:3375:ptlrpc_svcpt_stop_threads()) Process leaving 00000100:00000001:2.0:1773941498.790088:0:461:0:(service.c:3391:ptlrpc_stop_all_threads()) Process leaving 00000400:00000200:2.0:1773941498.790092:0:461:0:(lib-ptl.c:969:lnet_clear_lazy_portal()) clearing portal 15 lazy 00000400:00000001:2.0:1773941498.790096:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941498.790098:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941498.790100:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d87790. 00000400:00000200:2.0:1773941498.790103:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d035dac00 00000400:00000010:2.0:1773941498.790105:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d035dac00. 00000100:00000001:2.0:1773941498.790109:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1773941498.790111:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_cbd 00000100:00100000:2.0:1773941498.790119:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:2.0:1773941498.790121:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 0 buffers still posted 00000100:00000001:2.0:1773941498.790124:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:2.0:1773941498.790126:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941498.790128:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941498.790130:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203796e0. 00000400:00000200:2.0:1773941498.790139:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d21c4c240 00000400:00000010:2.0:1773941498.790141:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d21c4c240. 00000100:00000001:2.0:1773941498.790145:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1773941498.790146:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_cbd 00000100:00100000:2.0:1773941498.790154:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:2.0:1773941498.790156:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 0 buffers still posted 00000100:00000001:2.0:1773941498.790159:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:2.0:1773941498.790163:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1773941498.790166:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000010:2.0:1773941498.790171:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d18a18000. 00000100:00000010:2.0:1773941498.790177:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1081c800. 00000100:00000010:2.0:1773941498.790180:0:461:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff988d1977a800. 00000100:00000010:2.0:1773941498.790183:0:461:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff988d1977c800. 00000100:00000001:2.0:1773941498.790185:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1773941498.790187:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000010:2.0:1773941498.790190:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d18a1c000. 00000100:00000010:2.0:1773941498.790193:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1081e400. 00000100:00000010:2.0:1773941498.790196:0:461:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff988d0835d000. 00000100:00000010:2.0:1773941498.790199:0:461:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff988d09304800. 00000100:00000001:2.0:1773941498.790202:0:461:0:(nrs.c:1051:nrs_svcpt_cleanup_locked()) Process entered 00000100:00000001:2.0:1773941498.790205:0:461:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:2.0:1773941498.790207:0:461:0:(nrs.c:111:nrs_policy_stop_locked()) Process entered 00000100:00000001:2.0:1773941498.790209:0:461:0:(nrs.c:67:nrs_policy_stop0()) Process entered 00000100:00000010:2.0:1773941498.790211:0:461:0:(nrs_fifo.c:83:nrs_fifo_stop()) kfreed 'head': 40 at ffff988d209c1fc0. 00000100:00000001:2.0:1773941498.790226:0:461:0:(nrs.c:85:nrs_policy_stop0()) Process leaving 00000100:00000001:2.0:1773941498.790228:0:461:0:(nrs.c:147:nrs_policy_stop_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.790230:0:461:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:2.0:1773941498.790232:0:461:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d08276f00. 00000100:00000001:2.0:1773941498.790234:0:461:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:2.0:1773941498.790236:0:461:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:2.0:1773941498.790237:0:461:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d08276c00. 00000100:00000001:2.0:1773941498.790239:0:461:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:2.0:1773941498.790240:0:461:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:2.0:1773941498.790241:0:461:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d082763c0. 00000100:00000001:2.0:1773941498.790243:0:461:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:2.0:1773941498.790244:0:461:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:2.0:1773941498.790245:0:461:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d08276900. 00000100:00000001:2.0:1773941498.790248:0:461:0:(nrs.c:1082:nrs_svcpt_cleanup_locked()) Process leaving 00000100:00000001:2.0:1773941498.790254:0:461:0:(nrs.c:1051:nrs_svcpt_cleanup_locked()) Process entered 00000100:00000001:2.0:1773941498.790256:0:461:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:2.0:1773941498.790257:0:461:0:(nrs.c:111:nrs_policy_stop_locked()) Process entered 00000100:00000001:2.0:1773941498.790258:0:461:0:(nrs.c:67:nrs_policy_stop0()) Process entered 00000100:00000010:2.0:1773941498.790259:0:461:0:(nrs_fifo.c:83:nrs_fifo_stop()) kfreed 'head': 40 at ffff988d209c1a40. 00000100:00000001:2.0:1773941498.790261:0:461:0:(nrs.c:85:nrs_policy_stop0()) Process leaving 00000100:00000001:2.0:1773941498.790262:0:461:0:(nrs.c:147:nrs_policy_stop_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.790264:0:461:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:2.0:1773941498.790265:0:461:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d08276000. 00000100:00000001:2.0:1773941498.790267:0:461:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:2.0:1773941498.790269:0:461:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:2.0:1773941498.790270:0:461:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d08276a80. 00000100:00000001:2.0:1773941498.790272:0:461:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:2.0:1773941498.790273:0:461:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:2.0:1773941498.790274:0:461:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d0705b780. 00000100:00000001:2.0:1773941498.790277:0:461:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:2.0:1773941498.790279:0:461:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:2.0:1773941498.790280:0:461:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d1a040840. 00000100:00000001:2.0:1773941498.790282:0:461:0:(nrs.c:1082:nrs_svcpt_cleanup_locked()) Process leaving 00000020:00000010:2.0:1773941498.790438:0:461:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 3680 at ffff988d035d3000. 00000020:00000010:2.0:1773941498.790444:0:461:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff988d1081dc00. 00000100:00000010:2.0:1773941498.790476:0:461:0:(service.c:3818:ptlrpc_service_free()) kfreed 'array->paa_reqs_array': 12080 at ffff988d18b40000. 00000100:00000010:2.0:1773941498.790486:0:461:0:(service.c:3824:ptlrpc_service_free()) kfreed 'array->paa_reqs_count': 3020 at ffff988d035d2000. 00000100:00000010:2.0:1773941498.790489:0:461:0:(service.c:3818:ptlrpc_service_free()) kfreed 'array->paa_reqs_array': 12080 at ffff988d203a8000. 00000100:00000010:2.0:1773941498.790498:0:461:0:(service.c:3824:ptlrpc_service_free()) kfreed 'array->paa_reqs_count': 3020 at ffff988d035d0000. 00000100:00000010:2.0:1773941498.790502:0:461:0:(service.c:3830:ptlrpc_service_free()) kfreed 'svcpt': 784 at ffff988d1081e000. 00000100:00000010:2.0:1773941498.790506:0:461:0:(service.c:3830:ptlrpc_service_free()) kfreed 'svcpt': 784 at ffff988d1081d400. 00000100:00000010:2.0:1773941498.790509:0:461:0:(service.c:3836:ptlrpc_service_free()) kfreed 'svc': 360 at ffff988d07ebae00. 00000100:00000001:2.0:1773941498.790512:0:461:0:(service.c:3863:ptlrpc_unregister_service()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.790515:0:461:0:(service.c:3841:ptlrpc_unregister_service()) Process entered 00000100:00000200:2.0:1773941498.790516:0:461:0:(service.c:3843:ptlrpc_unregister_service()) ldlm_canceld: tearing down 00000100:00000001:2.0:1773941498.790520:0:461:0:(service.c:3384:ptlrpc_stop_all_threads()) Process entered 00000100:00000001:2.0:1773941498.790521:0:461:0:(service.c:3338:ptlrpc_svcpt_stop_threads()) Process entered 00000100:00000040:2.0:1773941498.790522:0:461:0:(service.c:3341:ptlrpc_svcpt_stop_threads()) Stopping threads for service ldlm_canceld 00000100:00000040:2.0:1773941498.790525:0:461:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cn #2 00000100:00000040:2.0:1773941498.790526:0:461:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cn #1 00000100:00000040:2.0:1773941498.790528:0:461:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cn #0 00000100:00000040:2.0:1773941498.790552:0:461:0:(service.c:3360:ptlrpc_svcpt_stop_threads()) waiting for stopping-thread ldlm_cn #2 00000020:00000010:1.0:1773941498.790562:0:14582:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d1a246c00. 00000020:00000010:1.0:1773941498.790572:0:14582:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d20938d80. 00000040:00000010:1.0:1773941498.790577:0:14582:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d1a246200. 00000020:00000010:1.0:1773941498.790581:0:14582:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d19779000. 00000020:00000010:1.0:1773941498.790586:0:14582:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff988d181f2840. 00000020:00000010:1.0:1773941498.790590:0:14582:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d02df8200. 80000000:00000010:1.0:1773941498.790595:0:14582:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d20938a20. 40000000:00000010:1.0:1773941498.790599:0:14582:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff988d03e7c4c0. 00008000:00000010:1.0:1773941498.790604:0:14582:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d0833a350. 00100000:00000010:1.0:1773941498.790608:0:14582:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff988d21759000. 00040000:00000010:1.0:1773941498.790613:0:14582:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d02cfd900. 00040000:00000010:1.0:1773941498.790616:0:14582:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff988d1a9ad800. 00040000:00000010:1.0:1773941498.790619:0:14582:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d0a32f800. 00000004:00000010:1.0:1773941498.790622:0:14582:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff988d1a9ae800. 00000004:00000010:1.0:1773941498.790624:0:14582:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff988d0834b000. 00000004:00000010:1.0:1773941498.790634:0:14582:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff988d19763800. 00002000:00000010:1.0:1773941498.790637:0:14582:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff988d035cc000. 00000004:00000010:1.0:1773941498.790640:0:14582:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff988d035cf400. 00000020:00000010:1.0:1773941498.790643:0:14582:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d1a246a00. 00000020:00000010:1.0:1773941498.790646:0:14582:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff988d03e7c0c0. 00000100:00000010:1.0:1773941498.790652:0:14582:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff988d03e7cb80. 00000100:00100000:1.0:1773941498.790654:0:14582:0:(service.c:3162:ptlrpc_main()) ldlm_cn00_000: service thread [ffff988d1a247800:14582] 0 exiting: rc = 1 00000020:00000010:1.0:1773941498.790705:0:14583:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d181fc400. 00000020:00000010:1.0:1773941498.790709:0:14583:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d2024ade0. 00000040:00000010:1.0:1773941498.790713:0:14583:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d181fd800. 00000020:00000010:1.0:1773941498.790716:0:14583:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d08996000. 00000020:00000010:1.0:1773941498.790720:0:14583:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff988d181f5a90. 00000020:00000010:1.0:1773941498.790723:0:14583:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d181fd600. 80000000:00000010:1.0:1773941498.790725:0:14583:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d2024a600. 40000000:00000010:1.0:1773941498.790727:0:14583:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff988d03e7c240. 00008000:00000010:1.0:1773941498.790729:0:14583:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d083391a8. 00100000:00000010:1.0:1773941498.790732:0:14583:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff988d08997800. 00040000:00000010:1.0:1773941498.790735:0:14583:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d02cfd800. 00040000:00000010:1.0:1773941498.790737:0:14583:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff988d08997000. 00040000:00000010:1.0:1773941498.790741:0:14583:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d181fc800. 00000004:00000010:1.0:1773941498.790744:0:14583:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff988d08994000. 00000004:00000010:1.0:1773941498.790746:0:14583:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff988d02bb9000. 00000004:00000010:1.0:1773941498.790749:0:14583:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff988d08993000. 00002000:00000010:1.0:1773941498.790751:0:14583:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff988d035ce400. 00000004:00000010:1.0:1773941498.790753:0:14583:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff988d035cc400. 00000020:00000010:1.0:1773941498.790756:0:14583:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0a130000. 00000020:00000010:1.0:1773941498.790761:0:14583:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff988d03e7c480. 00000100:00000010:1.0:1773941498.790766:0:14583:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff988d03e7cdc0. 00000100:00100000:1.0:1773941498.790769:0:14583:0:(service.c:3162:ptlrpc_main()) ldlm_cn00_001: service thread [ffff988d0642c200:14583] 1 exiting: rc = 0 00000020:00000010:1.0:1773941498.790806:0:14584:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d181fce00. 00000020:00000010:1.0:1773941498.790809:0:14584:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d054f3300. 00000040:00000010:1.0:1773941498.790811:0:14584:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d181fd000. 00000020:00000010:1.0:1773941498.790813:0:14584:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d08995800. 00000020:00000010:1.0:1773941498.790815:0:14584:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff988d181f0000. 00000020:00000010:1.0:1773941498.790824:0:14584:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d181fc200. 80000000:00000010:1.0:1773941498.790827:0:14584:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d054f3000. 40000000:00000010:1.0:1773941498.790829:0:14584:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff988d03e7c840. 00008000:00000010:1.0:1773941498.790831:0:14584:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d0833b880. 00100000:00000010:1.0:1773941498.790833:0:14584:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff988d08996800. 00040000:00000010:1.0:1773941498.790835:0:14584:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d02cfd200. 00040000:00000010:1.0:1773941498.790837:0:14584:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff988d08995000. 00040000:00000010:1.0:1773941498.790839:0:14584:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d181fc600. 00000004:00000010:1.0:1773941498.790842:0:14584:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff988d08990000. 00000004:00000010:1.0:1773941498.790845:0:14584:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff988d0396b000. 00000004:00000010:1.0:1773941498.790848:0:14584:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff988d08990800. 00002000:00000010:1.0:1773941498.790850:0:14584:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff988d1118b800. 00000004:00000010:1.0:1773941498.790852:0:14584:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff988d11188c00. 00000020:00000010:1.0:1773941498.790855:0:14584:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d181fc000. 00000020:00000010:1.0:1773941498.790859:0:14584:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff988d03e7cc40. 00000100:00000010:1.0:1773941498.790862:0:14584:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff988d03e7c800. 00000100:00100000:1.0:1773941498.790864:0:14584:0:(service.c:3162:ptlrpc_main()) ldlm_cn00_002: service thread [ffff988d0642d800:14584] 2 exiting: rc = 0 00000100:00000010:3.0:1773941498.793351:0:461:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff988d1a247800. 00000100:00000010:3.0:1773941498.793356:0:461:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff988d0642c200. 00000100:00000010:3.0:1773941498.793361:0:461:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff988d0642d800. 00000100:00000001:3.0:1773941498.793364:0:461:0:(service.c:3375:ptlrpc_svcpt_stop_threads()) Process leaving 00000100:00000001:3.0:1773941498.793365:0:461:0:(service.c:3338:ptlrpc_svcpt_stop_threads()) Process entered 00000100:00000040:3.0:1773941498.793366:0:461:0:(service.c:3341:ptlrpc_svcpt_stop_threads()) Stopping threads for service ldlm_canceld 00000100:00000040:3.0:1773941498.793368:0:461:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cn #2 00000100:00000040:3.0:1773941498.793370:0:461:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cn #1 00000100:00000040:3.0:1773941498.793371:0:461:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cn #0 00000100:00000040:3.0:1773941498.793381:0:461:0:(service.c:3360:ptlrpc_svcpt_stop_threads()) waiting for stopping-thread ldlm_cn #2 00000020:00000010:3.0:1773941498.793389:0:14586:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d0335c400. 00000020:00000010:3.0:1773941498.793392:0:14586:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d18350600. 00000040:00000010:3.0:1773941498.793395:0:14586:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d0335d400. 00000020:00000010:3.0:1773941498.793397:0:14586:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d08358000. 00000020:00000010:3.0:1773941498.793400:0:14586:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff988d03341e30. 00000020:00000010:3.0:1773941498.793403:0:14586:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d0335d800. 80000000:00000010:3.0:1773941498.793412:0:14586:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d18350360. 40000000:00000010:3.0:1773941498.793414:0:14586:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff988d02f4d000. 00008000:00000010:3.0:1773941498.793418:0:14586:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d107f3c08. 00100000:00000010:3.0:1773941498.793421:0:14586:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff988d08358800. 00040000:00000010:3.0:1773941498.793424:0:14586:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d02cba000. 00040000:00000010:3.0:1773941498.793426:0:14586:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff988d08359800. 00040000:00000010:3.0:1773941498.793429:0:14586:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d0335c800. 00000004:00000010:3.0:1773941498.793431:0:14586:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff988d0835e800. 00000004:00000010:3.0:1773941498.793434:0:14586:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff988d02d6f000. 00000004:00000010:3.0:1773941498.793437:0:14586:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff988d0835d800. 00002000:00000010:3.0:1773941498.793439:0:14586:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff988d054e1c00. 00000004:00000010:3.0:1773941498.793441:0:14586:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff988d054e2800. 00000020:00000010:3.0:1773941498.793445:0:14586:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0335ce00. 00000020:00000010:3.0:1773941498.793453:0:14586:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff988d02f4dd00. 00000100:00000010:3.0:1773941498.793458:0:14586:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff988d02f4dc80. 00000100:00100000:3.0:1773941498.793461:0:14586:0:(service.c:3162:ptlrpc_main()) ldlm_cn01_001: service thread [ffff988d04f8ae00:14586] 1 exiting: rc = 0 00000020:00000010:3.0:1773941498.793504:0:14587:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d02d19a00. 00000020:00000010:3.0:1773941498.793508:0:14587:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d08bbbf60. 00000040:00000010:3.0:1773941498.793510:0:14587:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d1a1c8c00. 00000020:00000010:3.0:1773941498.793512:0:14587:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d181c4000. 00000020:00000010:3.0:1773941498.793515:0:14587:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff988d05403758. 00000020:00000010:3.0:1773941498.793518:0:14587:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d1a1c8200. 80000000:00000010:3.0:1773941498.793520:0:14587:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d08bbba80. 40000000:00000010:3.0:1773941498.793522:0:14587:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff988d05437a00. 00008000:00000010:3.0:1773941498.793523:0:14587:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d08325fc8. 00100000:00000010:3.0:1773941498.793526:0:14587:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff988d181c7000. 00040000:00000010:3.0:1773941498.793528:0:14587:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d0a0dbb00. 00040000:00000010:3.0:1773941498.793531:0:14587:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff988d181c7800. 00040000:00000010:3.0:1773941498.793533:0:14587:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d1a1c8600. 00000004:00000010:3.0:1773941498.793535:0:14587:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff988d181c5800. 00000004:00000010:3.0:1773941498.793537:0:14587:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff988d08afc000. 00000004:00000010:3.0:1773941498.793541:0:14587:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff988d181c4800. 00002000:00000010:3.0:1773941498.793543:0:14587:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff988d035a7c00. 00000004:00000010:3.0:1773941498.793545:0:14587:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff988d035a4c00. 00000020:00000010:3.0:1773941498.793555:0:14587:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d02d18e00. 00000020:00000010:3.0:1773941498.793558:0:14587:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff988d05437740. 00000100:00000010:3.0:1773941498.793562:0:14587:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff988d05437180. 00000100:00100000:3.0:1773941498.793565:0:14587:0:(service.c:3162:ptlrpc_main()) ldlm_cn01_002: service thread [ffff988d181fd400:14587] 2 exiting: rc = 0 00000020:00000010:3.0:1773941498.793606:0:14585:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d02d18a00. 00000020:00000010:3.0:1773941498.793611:0:14585:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d08bbb600. 00000040:00000010:3.0:1773941498.793614:0:14585:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d02d18600. 00000020:00000010:3.0:1773941498.793617:0:14585:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d181c1800. 00000020:00000010:3.0:1773941498.793620:0:14585:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff988d05400508. 00000020:00000010:3.0:1773941498.793624:0:14585:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d02d18000. 80000000:00000010:3.0:1773941498.793627:0:14585:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d08bbb1e0. 40000000:00000010:3.0:1773941498.793630:0:14585:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff988d05437240. 00008000:00000010:3.0:1773941498.793632:0:14585:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d083266d8. 00100000:00000010:3.0:1773941498.793636:0:14585:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff988d181c1000. 00040000:00000010:3.0:1773941498.793639:0:14585:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d0a0db200. 00040000:00000010:3.0:1773941498.793641:0:14585:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff988d181c2000. 00040000:00000010:3.0:1773941498.793643:0:14585:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d02d18c00. 00000004:00000010:3.0:1773941498.793645:0:14585:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff988d181c0800. 00000004:00000010:3.0:1773941498.793647:0:14585:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff988d08aff000. 00000004:00000010:3.0:1773941498.793649:0:14585:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff988d181c0000. 00002000:00000010:3.0:1773941498.793651:0:14585:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff988d035a7400. 00000004:00000010:3.0:1773941498.793655:0:14585:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff988d035a5c00. 00000020:00000010:3.0:1773941498.793657:0:14585:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d02d18400. 00000020:00000010:3.0:1773941498.793659:0:14585:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff988d054378c0. 00000100:00000010:3.0:1773941498.793662:0:14585:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff988d05437fc0. 00000100:00100000:3.0:1773941498.793663:0:14585:0:(service.c:3162:ptlrpc_main()) ldlm_cn01_000: service thread [ffff988d0642d200:14585] 0 exiting: rc = 1 00000100:00000010:3.0:1773941498.793691:0:461:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff988d0642d200. 00000100:00000010:3.0:1773941498.793694:0:461:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff988d04f8ae00. 00000100:00000010:3.0:1773941498.793696:0:461:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff988d181fd400. 00000100:00000001:3.0:1773941498.793698:0:461:0:(service.c:3375:ptlrpc_svcpt_stop_threads()) Process leaving 00000100:00000001:3.0:1773941498.793699:0:461:0:(service.c:3391:ptlrpc_stop_all_threads()) Process leaving 00000400:00000200:3.0:1773941498.793701:0:461:0:(lib-ptl.c:969:lnet_clear_lazy_portal()) clearing portal 17 lazy 00000400:00000001:3.0:1773941498.793704:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.793705:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.793713:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c160. 00000400:00000200:3.0:1773941498.793715:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d063e4540 00000400:00000010:3.0:1773941498.793716:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d063e4540. 00000100:00000001:3.0:1773941498.793719:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.793720:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.793727:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.793729:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:3.0:1773941498.793731:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.793732:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.793733:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.793734:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1caa8. 00000400:00000200:3.0:1773941498.793736:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d063e46c0 00000400:00000010:3.0:1773941498.793737:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d063e46c0. 00000100:00000001:3.0:1773941498.793739:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.793740:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.793743:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.793745:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 62 buffers still posted 00000100:00000001:3.0:1773941498.793747:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.793749:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.793750:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.793751:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c108. 00000400:00000200:3.0:1773941498.793753:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d063e4d80 00000400:00000010:3.0:1773941498.793754:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d063e4d80. 00000100:00000001:3.0:1773941498.793756:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.793757:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.793763:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.793764:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 61 buffers still posted 00000100:00000001:3.0:1773941498.793765:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.793767:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.793767:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.793769:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1ca50. 00000400:00000200:3.0:1773941498.793772:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d08e343c0 00000400:00000010:3.0:1773941498.793773:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d08e343c0. 00000100:00000001:3.0:1773941498.793777:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.793778:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.793782:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.793784:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 60 buffers still posted 00000100:00000001:3.0:1773941498.793786:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.793793:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.793795:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.793796:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1cf78. 00000400:00000200:3.0:1773941498.793800:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d08e34840 00000400:00000010:3.0:1773941498.793801:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d08e34840. 00000100:00000001:3.0:1773941498.793805:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.793806:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.793811:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.793812:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 59 buffers still posted 00000100:00000001:3.0:1773941498.793814:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.793816:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.793817:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.793819:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c688. 00000400:00000200:3.0:1773941498.793821:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d08e34cc0 00000400:00000010:3.0:1773941498.793822:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d08e34cc0. 00000100:00000001:3.0:1773941498.793824:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.793826:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.793829:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.793831:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 58 buffers still posted 00000100:00000001:3.0:1773941498.793833:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.793836:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.793837:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.793839:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c478. 00000400:00000200:3.0:1773941498.793842:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d08e34f00 00000400:00000010:3.0:1773941498.793843:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d08e34f00. 00000100:00000001:3.0:1773941498.793846:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.793847:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.793854:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.793855:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 57 buffers still posted 00000100:00000001:3.0:1773941498.793858:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.793860:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.793862:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.793864:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c738. 00000400:00000200:3.0:1773941498.793868:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d08e34900 00000400:00000010:3.0:1773941498.793870:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d08e34900. 00000100:00000001:3.0:1773941498.793874:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.793876:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.793883:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.793892:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 56 buffers still posted 00000100:00000001:3.0:1773941498.793895:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.793898:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.793900:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.793902:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c0b0. 00000400:00000200:3.0:1773941498.793907:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d08e34e40 00000400:00000010:3.0:1773941498.793909:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d08e34e40. 00000100:00000001:3.0:1773941498.793912:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.793915:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.793923:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.793925:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 55 buffers still posted 00000100:00000001:3.0:1773941498.793928:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.793930:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.793931:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.793933:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1cdc0. 00000400:00000200:3.0:1773941498.793937:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d08e34300 00000400:00000010:3.0:1773941498.793938:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d08e34300. 00000100:00000001:3.0:1773941498.793942:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.793943:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.793949:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.793951:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 54 buffers still posted 00000100:00000001:3.0:1773941498.793953:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.793954:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.793955:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.793956:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c3c8. 00000400:00000200:3.0:1773941498.793959:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d08e34600 00000400:00000010:3.0:1773941498.793981:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d08e34600. 00000100:00000001:3.0:1773941498.793985:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.793986:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.793993:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.793995:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 53 buffers still posted 00000100:00000001:3.0:1773941498.793996:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.793998:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.793999:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794000:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c8f0. 00000400:00000200:3.0:1773941498.794003:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a033cc0 00000400:00000010:3.0:1773941498.794004:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a033cc0. 00000100:00000001:3.0:1773941498.794008:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794015:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794020:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794022:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 52 buffers still posted 00000100:00000001:3.0:1773941498.794025:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794027:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794029:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794031:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1cd68. 00000400:00000200:3.0:1773941498.794034:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a0333c0 00000400:00000010:3.0:1773941498.794035:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a0333c0. 00000100:00000001:3.0:1773941498.794038:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794039:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794046:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794047:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 51 buffers still posted 00000100:00000001:3.0:1773941498.794050:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794051:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794053:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794054:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1cf20. 00000400:00000200:3.0:1773941498.794058:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d20863780 00000400:00000010:3.0:1773941498.794060:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d20863780. 00000100:00000001:3.0:1773941498.794063:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794064:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794071:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794072:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 50 buffers still posted 00000100:00000001:3.0:1773941498.794074:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794075:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794077:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794078:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1cb58. 00000400:00000200:3.0:1773941498.794081:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d20863f00 00000400:00000010:3.0:1773941498.794082:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d20863f00. 00000100:00000001:3.0:1773941498.794086:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794087:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794094:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794095:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 49 buffers still posted 00000100:00000001:3.0:1773941498.794098:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794100:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794101:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794102:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c6e0. 00000400:00000200:3.0:1773941498.794105:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d208639c0 00000400:00000010:3.0:1773941498.794113:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d208639c0. 00000100:00000001:3.0:1773941498.794116:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794117:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794124:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794126:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 48 buffers still posted 00000100:00000001:3.0:1773941498.794128:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794130:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794131:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794132:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1ccb8. 00000400:00000200:3.0:1773941498.794136:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d20863480 00000400:00000010:3.0:1773941498.794137:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d20863480. 00000100:00000001:3.0:1773941498.794141:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794142:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794146:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794148:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 47 buffers still posted 00000100:00000001:3.0:1773941498.794150:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794151:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794152:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794154:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c000. 00000400:00000200:3.0:1773941498.794157:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d20863900 00000400:00000010:3.0:1773941498.794158:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d20863900. 00000100:00000001:3.0:1773941498.794161:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794162:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794167:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794168:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 46 buffers still posted 00000100:00000001:3.0:1773941498.794171:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794172:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794173:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794175:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1cc60. 00000400:00000200:3.0:1773941498.794177:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d20863540 00000400:00000010:3.0:1773941498.794178:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d20863540. 00000100:00000001:3.0:1773941498.794181:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794183:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794187:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794189:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 45 buffers still posted 00000100:00000001:3.0:1773941498.794190:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794192:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794198:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794200:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c1b8. 00000400:00000200:3.0:1773941498.794202:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d20863a80 00000400:00000010:3.0:1773941498.794203:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d20863a80. 00000100:00000001:3.0:1773941498.794207:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794208:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794211:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794212:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 44 buffers still posted 00000100:00000001:3.0:1773941498.794213:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794226:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794228:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794230:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1ce18. 00000400:00000200:3.0:1773941498.794232:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d20863e40 00000400:00000010:3.0:1773941498.794233:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d20863e40. 00000100:00000001:3.0:1773941498.794236:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794237:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794241:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794242:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 43 buffers still posted 00000100:00000001:3.0:1773941498.794245:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794247:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794248:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794249:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1cb00. 00000400:00000200:3.0:1773941498.794251:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d20863840 00000400:00000010:3.0:1773941498.794253:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d20863840. 00000100:00000001:3.0:1773941498.794255:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794256:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794261:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794262:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 42 buffers still posted 00000100:00000001:3.0:1773941498.794264:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794265:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794267:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794268:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c898. 00000400:00000200:3.0:1773941498.794270:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d20863180 00000400:00000010:3.0:1773941498.794271:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d20863180. 00000100:00000001:3.0:1773941498.794273:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794274:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794278:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794280:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 41 buffers still posted 00000100:00000001:3.0:1773941498.794288:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794290:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794292:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794294:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1ce70. 00000400:00000200:3.0:1773941498.794296:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d20863300 00000400:00000010:3.0:1773941498.794297:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d20863300. 00000100:00000001:3.0:1773941498.794299:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794300:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794306:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794308:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 40 buffers still posted 00000100:00000001:3.0:1773941498.794309:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794311:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794313:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794314:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c318. 00000400:00000200:3.0:1773941498.794316:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a03c840 00000400:00000010:3.0:1773941498.794317:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a03c840. 00000100:00000001:3.0:1773941498.794320:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794321:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794327:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794329:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 39 buffers still posted 00000100:00000001:3.0:1773941498.794331:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794332:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794333:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794334:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d875d8. 00000400:00000200:3.0:1773941498.794337:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a03c240 00000400:00000010:3.0:1773941498.794338:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a03c240. 00000100:00000001:3.0:1773941498.794340:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794342:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794346:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794347:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 38 buffers still posted 00000100:00000001:3.0:1773941498.794349:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794351:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794352:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794353:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d87840. 00000400:00000200:3.0:1773941498.794356:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a03c600 00000400:00000010:3.0:1773941498.794357:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a03c600. 00000100:00000001:3.0:1773941498.794359:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794360:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794368:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794370:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 37 buffers still posted 00000100:00000001:3.0:1773941498.794371:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794373:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794374:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794375:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d87370. 00000400:00000200:3.0:1773941498.794376:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a03ccc0 00000400:00000010:3.0:1773941498.794378:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a03ccc0. 00000100:00000001:3.0:1773941498.794379:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794380:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794382:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794383:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 36 buffers still posted 00000100:00000001:3.0:1773941498.794384:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794385:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794386:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794387:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d877e8. 00000400:00000200:3.0:1773941498.794388:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a03c180 00000400:00000010:3.0:1773941498.794389:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a03c180. 00000100:00000001:3.0:1773941498.794390:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794391:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794394:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794395:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 35 buffers still posted 00000100:00000001:3.0:1773941498.794396:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794397:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794398:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794399:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d87f78. 00000400:00000200:3.0:1773941498.794401:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a03cf00 00000400:00000010:3.0:1773941498.794402:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a03cf00. 00000100:00000001:3.0:1773941498.794403:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794404:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794409:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794410:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 34 buffers still posted 00000100:00000001:3.0:1773941498.794412:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794414:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794415:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794417:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d87688. 00000400:00000200:3.0:1773941498.794419:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04c810c0 00000400:00000010:3.0:1773941498.794420:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d04c810c0. 00000100:00000001:3.0:1773941498.794429:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794430:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794434:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794436:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 33 buffers still posted 00000100:00000001:3.0:1773941498.794437:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794439:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794440:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794441:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d87478. 00000400:00000200:3.0:1773941498.794443:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04c81840 00000400:00000010:3.0:1773941498.794444:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d04c81840. 00000100:00000001:3.0:1773941498.794446:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794447:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794451:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794453:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 32 buffers still posted 00000100:00000001:3.0:1773941498.794454:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794455:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794456:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794457:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d87738. 00000400:00000200:3.0:1773941498.794459:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d06510780 00000400:00000010:3.0:1773941498.794460:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d06510780. 00000100:00000001:3.0:1773941498.794463:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794464:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794467:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794469:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 31 buffers still posted 00000100:00000001:3.0:1773941498.794470:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794471:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794473:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794474:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d870b0. 00000400:00000200:3.0:1773941498.794476:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d06510b40 00000400:00000010:3.0:1773941498.794478:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d06510b40. 00000100:00000001:3.0:1773941498.794481:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794482:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794487:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794488:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 30 buffers still posted 00000100:00000001:3.0:1773941498.794491:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794493:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794494:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794495:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d87dc0. 00000400:00000200:3.0:1773941498.794504:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d06510a80 00000400:00000010:3.0:1773941498.794505:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d06510a80. 00000100:00000001:3.0:1773941498.794509:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794510:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794514:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794516:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 29 buffers still posted 00000100:00000001:3.0:1773941498.794519:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794520:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794521:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794522:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d873c8. 00000400:00000200:3.0:1773941498.794525:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d065103c0 00000400:00000010:3.0:1773941498.794527:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d065103c0. 00000100:00000001:3.0:1773941498.794530:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794531:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794537:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794539:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 28 buffers still posted 00000100:00000001:3.0:1773941498.794541:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794543:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794544:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794545:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d878f0. 00000400:00000200:3.0:1773941498.794548:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d06510f00 00000400:00000010:3.0:1773941498.794549:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d06510f00. 00000100:00000001:3.0:1773941498.794552:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794553:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794557:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794559:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 27 buffers still posted 00000100:00000001:3.0:1773941498.794561:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794563:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794564:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794565:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d87d68. 00000400:00000200:3.0:1773941498.794568:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d065106c0 00000400:00000010:3.0:1773941498.794569:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d065106c0. 00000100:00000001:3.0:1773941498.794573:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794574:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794583:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794585:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 26 buffers still posted 00000100:00000001:3.0:1773941498.794589:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794598:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794600:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794602:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d87f20. 00000400:00000200:3.0:1773941498.794606:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d065100c0 00000400:00000010:3.0:1773941498.794608:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d065100c0. 00000100:00000001:3.0:1773941498.794613:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794615:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794621:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794624:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 25 buffers still posted 00000100:00000001:3.0:1773941498.794627:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794629:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794631:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794633:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d87b58. 00000400:00000200:3.0:1773941498.794636:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d06510cc0 00000400:00000010:3.0:1773941498.794638:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d06510cc0. 00000100:00000001:3.0:1773941498.794641:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794642:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794648:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794650:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 24 buffers still posted 00000100:00000001:3.0:1773941498.794652:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794654:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794655:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794657:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d876e0. 00000400:00000200:3.0:1773941498.794660:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d06510d80 00000400:00000010:3.0:1773941498.794661:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d06510d80. 00000100:00000001:3.0:1773941498.794663:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794664:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794668:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794670:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 23 buffers still posted 00000100:00000001:3.0:1773941498.794671:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794673:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794674:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794675:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d87cb8. 00000400:00000200:3.0:1773941498.794678:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d06510e40 00000400:00000010:3.0:1773941498.794679:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d06510e40. 00000100:00000001:3.0:1773941498.794682:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794683:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794687:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794694:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 22 buffers still posted 00000100:00000001:3.0:1773941498.794696:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794699:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794700:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794701:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d87000. 00000400:00000200:3.0:1773941498.794704:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d06510480 00000400:00000010:3.0:1773941498.794705:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d06510480. 00000100:00000001:3.0:1773941498.794708:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794709:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794715:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794716:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 21 buffers still posted 00000100:00000001:3.0:1773941498.794717:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794719:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794720:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794721:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d87c60. 00000400:00000200:3.0:1773941498.794723:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d06510300 00000400:00000010:3.0:1773941498.794725:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d06510300. 00000100:00000001:3.0:1773941498.794727:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794729:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794734:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794735:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 20 buffers still posted 00000100:00000001:3.0:1773941498.794737:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794738:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794739:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794740:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d871b8. 00000400:00000200:3.0:1773941498.794743:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d06510180 00000400:00000010:3.0:1773941498.794744:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d06510180. 00000100:00000001:3.0:1773941498.794747:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794748:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794753:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794754:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 19 buffers still posted 00000100:00000001:3.0:1773941498.794756:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794757:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794759:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794760:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d87e18. 00000400:00000200:3.0:1773941498.794763:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d06510900 00000400:00000010:3.0:1773941498.794764:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d06510900. 00000100:00000001:3.0:1773941498.794767:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794773:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794776:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794777:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 18 buffers still posted 00000100:00000001:3.0:1773941498.794778:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794779:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794780:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794781:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d87b00. 00000400:00000200:3.0:1773941498.794783:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d06510240 00000400:00000010:3.0:1773941498.794784:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d06510240. 00000100:00000001:3.0:1773941498.794786:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794787:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794790:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794791:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 17 buffers still posted 00000100:00000001:3.0:1773941498.794792:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794794:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794795:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794796:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d87898. 00000400:00000200:3.0:1773941498.794797:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d06510000 00000400:00000010:3.0:1773941498.794798:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d06510000. 00000100:00000001:3.0:1773941498.794799:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794800:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794803:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794805:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 16 buffers still posted 00000100:00000001:3.0:1773941498.794806:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794807:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794808:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794810:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d87e70. 00000400:00000200:3.0:1773941498.794812:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d06510540 00000400:00000010:3.0:1773941498.794813:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d06510540. 00000100:00000001:3.0:1773941498.794815:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794816:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794820:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794821:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 15 buffers still posted 00000100:00000001:3.0:1773941498.794823:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794824:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794825:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794826:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d87318. 00000400:00000200:3.0:1773941498.794828:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d06510c00 00000400:00000010:3.0:1773941498.794836:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d06510c00. 00000100:00000001:3.0:1773941498.794838:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794839:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794843:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794844:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 14 buffers still posted 00000100:00000001:3.0:1773941498.794846:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794847:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794848:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794849:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d879f8. 00000400:00000200:3.0:1773941498.794851:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d2081ad80 00000400:00000010:3.0:1773941498.794852:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d2081ad80. 00000100:00000001:3.0:1773941498.794854:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794855:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794860:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794862:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 13 buffers still posted 00000100:00000001:3.0:1773941498.794864:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794866:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794868:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794870:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d87630. 00000400:00000200:3.0:1773941498.794873:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d2081a0c0 00000400:00000010:3.0:1773941498.794875:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d2081a0c0. 00000100:00000001:3.0:1773941498.794878:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794879:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794884:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794886:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 12 buffers still posted 00000100:00000001:3.0:1773941498.794888:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794891:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794892:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794894:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d87c08. 00000400:00000200:3.0:1773941498.794897:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d2081a480 00000400:00000010:3.0:1773941498.794900:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d2081a480. 00000100:00000001:3.0:1773941498.794903:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794905:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794912:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794914:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 11 buffers still posted 00000100:00000001:3.0:1773941498.794917:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794919:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794926:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794927:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d874d0. 00000400:00000200:3.0:1773941498.794930:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d034cc240 00000400:00000010:3.0:1773941498.794931:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d034cc240. 00000100:00000001:3.0:1773941498.794934:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794935:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794938:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794939:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 10 buffers still posted 00000100:00000001:3.0:1773941498.794941:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794942:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794943:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794944:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d879a0. 00000400:00000200:3.0:1773941498.794947:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d21dd1f00 00000400:00000010:3.0:1773941498.794948:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d21dd1f00. 00000100:00000001:3.0:1773941498.794950:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794951:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794956:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.794958:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 9 buffers still posted 00000100:00000001:3.0:1773941498.794959:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.794979:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.794980:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.794981:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d87948. 00000400:00000200:3.0:1773941498.794985:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d02dcbb40 00000400:00000010:3.0:1773941498.794986:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d02dcbb40. 00000100:00000001:3.0:1773941498.794992:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.794994:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.794999:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795000:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 8 buffers still posted 00000100:00000001:3.0:1773941498.795002:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795003:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795004:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795006:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d872c0. 00000400:00000200:3.0:1773941498.795008:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d2030bb40 00000400:00000010:3.0:1773941498.795009:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d2030bb40. 00000100:00000001:3.0:1773941498.795012:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795013:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795018:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795020:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 7 buffers still posted 00000100:00000001:3.0:1773941498.795027:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795029:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795030:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795032:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d87ec8. 00000400:00000200:3.0:1773941498.795034:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d2030b240 00000400:00000010:3.0:1773941498.795035:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d2030b240. 00000100:00000001:3.0:1773941498.795038:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795039:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795043:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795044:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 6 buffers still posted 00000100:00000001:3.0:1773941498.795047:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795048:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795049:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795050:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d87058. 00000400:00000200:3.0:1773941498.795053:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d2030b000 00000400:00000010:3.0:1773941498.795054:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d2030b000. 00000100:00000001:3.0:1773941498.795057:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795058:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795062:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795064:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 5 buffers still posted 00000100:00000001:3.0:1773941498.795066:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795067:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795068:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795070:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d87268. 00000400:00000200:3.0:1773941498.795072:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d2030ba80 00000400:00000010:3.0:1773941498.795073:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d2030ba80. 00000100:00000001:3.0:1773941498.795076:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795077:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795081:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795083:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 4 buffers still posted 00000100:00000001:3.0:1773941498.795085:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795086:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795087:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795089:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d87528. 00000400:00000200:3.0:1773941498.795091:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d2030b300 00000400:00000010:3.0:1773941498.795092:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d2030b300. 00000100:00000001:3.0:1773941498.795095:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795096:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795103:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795105:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 3 buffers still posted 00000100:00000001:3.0:1773941498.795106:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795108:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795109:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795110:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d87210. 00000400:00000200:3.0:1773941498.795113:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d2030bc00 00000400:00000010:3.0:1773941498.795114:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d2030bc00. 00000100:00000001:3.0:1773941498.795117:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795118:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795122:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795123:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 2 buffers still posted 00000100:00000001:3.0:1773941498.795125:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795127:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795128:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795129:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d87bb0. 00000400:00000200:3.0:1773941498.795132:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d2030b480 00000400:00000010:3.0:1773941498.795134:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d2030b480. 00000100:00000001:3.0:1773941498.795137:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795138:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795143:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795145:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 1 buffers still posted 00000100:00000001:3.0:1773941498.795147:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795149:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795150:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795151:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d21d87420. 00000400:00000200:3.0:1773941498.795154:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d2030b780 00000400:00000010:3.0:1773941498.795155:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d2030b780. 00000100:00000001:3.0:1773941498.795159:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795160:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795165:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795167:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 0 buffers still posted 00000100:00000001:3.0:1773941498.795169:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795170:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795171:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795173:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e268. 00000400:00000200:3.0:1773941498.795176:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18807cc0 00000400:00000010:3.0:1773941498.795177:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18807cc0. 00000100:00000001:3.0:1773941498.795185:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795186:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795191:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795193:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:3.0:1773941498.795195:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795197:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795198:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795200:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e528. 00000400:00000200:3.0:1773941498.795202:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18807780 00000400:00000010:3.0:1773941498.795204:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18807780. 00000100:00000001:3.0:1773941498.795207:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795208:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795212:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795213:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 62 buffers still posted 00000100:00000001:3.0:1773941498.795227:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795229:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795230:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795231:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e210. 00000400:00000200:3.0:1773941498.795235:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18807000 00000400:00000010:3.0:1773941498.795236:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18807000. 00000100:00000001:3.0:1773941498.795238:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795239:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795245:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795246:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 61 buffers still posted 00000100:00000001:3.0:1773941498.795248:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795250:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795251:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795252:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380ebb0. 00000400:00000200:3.0:1773941498.795256:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18807240 00000400:00000010:3.0:1773941498.795257:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18807240. 00000100:00000001:3.0:1773941498.795260:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795261:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795268:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795270:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 60 buffers still posted 00000100:00000001:3.0:1773941498.795272:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795273:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795275:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795276:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e420. 00000400:00000200:3.0:1773941498.795284:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18807e40 00000400:00000010:3.0:1773941498.795286:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18807e40. 00000100:00000001:3.0:1773941498.795289:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795290:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795295:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795297:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 59 buffers still posted 00000100:00000001:3.0:1773941498.795300:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795301:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795302:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795303:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e790. 00000400:00000200:3.0:1773941498.795306:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18807600 00000400:00000010:3.0:1773941498.795308:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18807600. 00000100:00000001:3.0:1773941498.795311:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795312:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795316:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795317:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 58 buffers still posted 00000100:00000001:3.0:1773941498.795319:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795320:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795321:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795322:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e370. 00000400:00000200:3.0:1773941498.795325:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18807480 00000400:00000010:3.0:1773941498.795326:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18807480. 00000100:00000001:3.0:1773941498.795329:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795330:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795334:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795336:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 57 buffers still posted 00000100:00000001:3.0:1773941498.795338:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795339:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795340:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795341:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e840. 00000400:00000200:3.0:1773941498.795343:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d188079c0 00000400:00000010:3.0:1773941498.795344:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d188079c0. 00000100:00000001:3.0:1773941498.795346:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795347:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795353:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795355:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 56 buffers still posted 00000100:00000001:3.0:1773941498.795356:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795362:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795363:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795364:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e7e8. 00000400:00000200:3.0:1773941498.795367:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18807180 00000400:00000010:3.0:1773941498.795369:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18807180. 00000100:00000001:3.0:1773941498.795372:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795373:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795379:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795380:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 55 buffers still posted 00000100:00000001:3.0:1773941498.795382:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795384:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795385:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795387:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e5d8. 00000400:00000200:3.0:1773941498.795389:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d188073c0 00000400:00000010:3.0:1773941498.795390:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d188073c0. 00000100:00000001:3.0:1773941498.795393:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795394:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795400:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795401:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 54 buffers still posted 00000100:00000001:3.0:1773941498.795403:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795405:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795405:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795407:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e580. 00000400:00000200:3.0:1773941498.795410:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18807a80 00000400:00000010:3.0:1773941498.795411:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18807a80. 00000100:00000001:3.0:1773941498.795413:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795414:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795417:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795418:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 53 buffers still posted 00000100:00000001:3.0:1773941498.795419:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795420:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795421:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795422:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380ed10. 00000400:00000200:3.0:1773941498.795424:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18807c00 00000400:00000010:3.0:1773941498.795425:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18807c00. 00000100:00000001:3.0:1773941498.795427:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795428:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795432:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795438:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 52 buffers still posted 00000100:00000001:3.0:1773941498.795439:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795441:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795442:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795443:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e160. 00000400:00000200:3.0:1773941498.795446:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18807d80 00000400:00000010:3.0:1773941498.795448:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18807d80. 00000100:00000001:3.0:1773941498.795450:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795450:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795453:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795455:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 51 buffers still posted 00000100:00000001:3.0:1773941498.795456:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795457:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795458:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795460:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380eaa8. 00000400:00000200:3.0:1773941498.795462:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a9ddb40 00000400:00000010:3.0:1773941498.795463:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a9ddb40. 00000100:00000001:3.0:1773941498.795465:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795467:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795471:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795472:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 50 buffers still posted 00000100:00000001:3.0:1773941498.795473:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795474:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795475:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795477:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e108. 00000400:00000200:3.0:1773941498.795479:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a9dd240 00000400:00000010:3.0:1773941498.795481:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a9dd240. 00000100:00000001:3.0:1773941498.795483:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795484:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795488:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795489:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 49 buffers still posted 00000100:00000001:3.0:1773941498.795490:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795491:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795492:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795493:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380ea50. 00000400:00000200:3.0:1773941498.795494:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a9ddcc0 00000400:00000010:3.0:1773941498.795495:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a9ddcc0. 00000100:00000001:3.0:1773941498.795497:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795502:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795506:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795507:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 48 buffers still posted 00000100:00000001:3.0:1773941498.795509:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795511:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795512:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795513:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380ef78. 00000400:00000200:3.0:1773941498.795515:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a9dd780 00000400:00000010:3.0:1773941498.795516:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a9dd780. 00000100:00000001:3.0:1773941498.795518:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795519:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795526:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795528:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 47 buffers still posted 00000100:00000001:3.0:1773941498.795529:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795530:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795531:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795532:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e688. 00000400:00000200:3.0:1773941498.795534:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a9dd0c0 00000400:00000010:3.0:1773941498.795535:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a9dd0c0. 00000100:00000001:3.0:1773941498.795537:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795539:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795542:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795544:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 46 buffers still posted 00000100:00000001:3.0:1773941498.795545:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795546:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795547:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795549:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e478. 00000400:00000200:3.0:1773941498.795551:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a9dd3c0 00000400:00000010:3.0:1773941498.795553:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a9dd3c0. 00000100:00000001:3.0:1773941498.795556:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795557:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795560:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795561:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 45 buffers still posted 00000100:00000001:3.0:1773941498.795562:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795563:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795564:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795566:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203791b8. 00000400:00000200:3.0:1773941498.795567:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a9dda80 00000400:00000010:3.0:1773941498.795573:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a9dda80. 00000100:00000001:3.0:1773941498.795575:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795576:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795581:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795582:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 44 buffers still posted 00000100:00000001:3.0:1773941498.795584:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795586:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795587:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795589:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379c60. 00000400:00000200:3.0:1773941498.795591:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a9dd540 00000400:00000010:3.0:1773941498.795593:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a9dd540. 00000100:00000001:3.0:1773941498.795595:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795597:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795601:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795603:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 43 buffers still posted 00000100:00000001:3.0:1773941498.795605:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795607:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795609:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795611:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379e18. 00000400:00000200:3.0:1773941498.795613:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a9dd180 00000400:00000010:3.0:1773941498.795616:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a9dd180. 00000100:00000001:3.0:1773941498.795618:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795619:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795624:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795626:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 42 buffers still posted 00000100:00000001:3.0:1773941498.795628:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795630:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795631:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795633:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379b00. 00000400:00000200:3.0:1773941498.795636:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a9dd840 00000400:00000010:3.0:1773941498.795637:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a9dd840. 00000100:00000001:3.0:1773941498.795638:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795639:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795642:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795643:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 41 buffers still posted 00000100:00000001:3.0:1773941498.795644:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795646:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795652:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795653:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379898. 00000400:00000200:3.0:1773941498.795654:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a9dd6c0 00000400:00000010:3.0:1773941498.795655:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a9dd6c0. 00000100:00000001:3.0:1773941498.795657:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795658:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795661:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795662:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 40 buffers still posted 00000100:00000001:3.0:1773941498.795663:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795664:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795665:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795666:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379e70. 00000400:00000200:3.0:1773941498.795668:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a9ddf00 00000400:00000010:3.0:1773941498.795669:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a9ddf00. 00000100:00000001:3.0:1773941498.795670:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795671:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795674:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795675:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 39 buffers still posted 00000100:00000001:3.0:1773941498.795676:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795677:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795677:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795678:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379318. 00000400:00000200:3.0:1773941498.795679:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a9dde40 00000400:00000010:3.0:1773941498.795680:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a9dde40. 00000100:00000001:3.0:1773941498.795681:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795682:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795684:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795685:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 38 buffers still posted 00000100:00000001:3.0:1773941498.795686:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795687:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795688:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795689:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203799f8. 00000400:00000200:3.0:1773941498.795690:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a9ddc00 00000400:00000010:3.0:1773941498.795691:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a9ddc00. 00000100:00000001:3.0:1773941498.795692:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795693:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795695:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795696:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 37 buffers still posted 00000100:00000001:3.0:1773941498.795701:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795703:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795704:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795705:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379630. 00000400:00000200:3.0:1773941498.795706:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a9dd300 00000400:00000010:3.0:1773941498.795707:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a9dd300. 00000100:00000001:3.0:1773941498.795708:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795709:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795712:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795713:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 36 buffers still posted 00000100:00000001:3.0:1773941498.795714:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795715:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795715:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795716:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379c08. 00000400:00000200:3.0:1773941498.795717:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a9dd9c0 00000400:00000010:3.0:1773941498.795718:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a9dd9c0. 00000100:00000001:3.0:1773941498.795719:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795720:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795722:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795723:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 35 buffers still posted 00000100:00000001:3.0:1773941498.795724:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795725:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795726:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795727:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203794d0. 00000400:00000200:3.0:1773941498.795728:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a9ddd80 00000400:00000010:3.0:1773941498.795729:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a9ddd80. 00000100:00000001:3.0:1773941498.795730:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795730:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795733:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795734:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 34 buffers still posted 00000100:00000001:3.0:1773941498.795734:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795735:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795736:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795737:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203799a0. 00000400:00000200:3.0:1773941498.795738:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a9dd900 00000400:00000010:3.0:1773941498.795739:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a9dd900. 00000100:00000001:3.0:1773941498.795740:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795741:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795748:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795749:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 33 buffers still posted 00000100:00000001:3.0:1773941498.795750:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795752:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795753:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795754:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379948. 00000400:00000200:3.0:1773941498.795755:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a9dd600 00000400:00000010:3.0:1773941498.795756:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a9dd600. 00000100:00000001:3.0:1773941498.795757:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795758:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795761:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795762:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 32 buffers still posted 00000100:00000001:3.0:1773941498.795762:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795764:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795764:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795765:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203792c0. 00000400:00000200:3.0:1773941498.795766:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18b98840 00000400:00000010:3.0:1773941498.795767:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18b98840. 00000100:00000001:3.0:1773941498.795768:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795769:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795772:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795772:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 31 buffers still posted 00000100:00000001:3.0:1773941498.795773:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795774:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795775:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795776:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379ec8. 00000400:00000200:3.0:1773941498.795777:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18b983c0 00000400:00000010:3.0:1773941498.795778:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18b983c0. 00000100:00000001:3.0:1773941498.795779:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795780:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795782:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795783:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 30 buffers still posted 00000100:00000001:3.0:1773941498.795784:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795785:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795786:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795787:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379058. 00000400:00000200:3.0:1773941498.795788:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18b98540 00000400:00000010:3.0:1773941498.795789:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18b98540. 00000100:00000001:3.0:1773941498.795794:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795795:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795799:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795800:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 29 buffers still posted 00000100:00000001:3.0:1773941498.795801:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795802:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795802:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795803:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379268. 00000400:00000200:3.0:1773941498.795804:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18b98480 00000400:00000010:3.0:1773941498.795805:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18b98480. 00000100:00000001:3.0:1773941498.795807:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795808:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795810:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795811:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 28 buffers still posted 00000100:00000001:3.0:1773941498.795812:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795813:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795813:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795814:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379528. 00000400:00000200:3.0:1773941498.795815:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18b98c00 00000400:00000010:3.0:1773941498.795816:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18b98c00. 00000100:00000001:3.0:1773941498.795817:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795818:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795821:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795822:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 27 buffers still posted 00000100:00000001:3.0:1773941498.795822:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795823:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795824:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795825:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379210. 00000400:00000200:3.0:1773941498.795826:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18b98900 00000400:00000010:3.0:1773941498.795827:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18b98900. 00000100:00000001:3.0:1773941498.795828:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795829:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795831:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795832:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 26 buffers still posted 00000100:00000001:3.0:1773941498.795833:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795834:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795835:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795836:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379bb0. 00000400:00000200:3.0:1773941498.795842:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18b98780 00000400:00000010:3.0:1773941498.795843:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18b98780. 00000100:00000001:3.0:1773941498.795844:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795845:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795848:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795849:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 25 buffers still posted 00000100:00000001:3.0:1773941498.795850:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795850:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795851:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795852:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379420. 00000400:00000200:3.0:1773941498.795853:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18b986c0 00000400:00000010:3.0:1773941498.795854:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18b986c0. 00000100:00000001:3.0:1773941498.795855:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795856:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795860:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795861:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 24 buffers still posted 00000100:00000001:3.0:1773941498.795863:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795864:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795865:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795867:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379790. 00000400:00000200:3.0:1773941498.795869:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18b989c0 00000400:00000010:3.0:1773941498.795870:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18b989c0. 00000100:00000001:3.0:1773941498.795872:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795873:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795877:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795878:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 23 buffers still posted 00000100:00000001:3.0:1773941498.795879:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795881:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795882:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795884:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379370. 00000400:00000200:3.0:1773941498.795886:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18b98a80 00000400:00000010:3.0:1773941498.795888:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18b98a80. 00000100:00000001:3.0:1773941498.795890:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795891:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795898:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795900:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 22 buffers still posted 00000100:00000001:3.0:1773941498.795902:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795910:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795912:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795913:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379840. 00000400:00000200:3.0:1773941498.795916:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18b980c0 00000400:00000010:3.0:1773941498.795919:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18b980c0. 00000100:00000001:3.0:1773941498.795922:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795923:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795927:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795928:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 21 buffers still posted 00000100:00000001:3.0:1773941498.795930:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795932:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795933:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795934:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203797e8. 00000400:00000200:3.0:1773941498.795936:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18b98d80 00000400:00000010:3.0:1773941498.795937:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18b98d80. 00000100:00000001:3.0:1773941498.795939:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795940:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.795943:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.795944:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 20 buffers still posted 00000100:00000001:3.0:1773941498.795945:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.795946:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.795947:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.795949:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:3.0:1773941498.795952:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18b98180 00000400:00000010:3.0:1773941498.795953:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18b98180. 00000100:00000001:3.0:1773941498.795956:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.795957:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.796062:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.796063:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 19 buffers still posted 00000100:00000001:3.0:1773941498.796065:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.796066:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.796067:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.796069:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379580. 00000400:00000200:3.0:1773941498.796072:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18b98000 00000400:00000010:3.0:1773941498.796074:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18b98000. 00000100:00000001:3.0:1773941498.796077:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.796078:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.796084:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.796091:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 18 buffers still posted 00000100:00000001:3.0:1773941498.796093:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.796095:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.796096:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.796098:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379d10. 00000400:00000200:3.0:1773941498.796101:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18b98600 00000400:00000010:3.0:1773941498.796102:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18b98600. 00000100:00000001:3.0:1773941498.796106:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.796107:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.796113:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.796114:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 17 buffers still posted 00000100:00000001:3.0:1773941498.796117:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.796118:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.796119:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.796120:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379160. 00000400:00000200:3.0:1773941498.796123:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18b98f00 00000400:00000010:3.0:1773941498.796125:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18b98f00. 00000100:00000001:3.0:1773941498.796128:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.796128:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.796134:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.796136:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 16 buffers still posted 00000100:00000001:3.0:1773941498.796138:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.796139:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.796140:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.796142:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379aa8. 00000400:00000200:3.0:1773941498.796144:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18b98300 00000400:00000010:3.0:1773941498.796146:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18b98300. 00000100:00000001:3.0:1773941498.796149:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.796150:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.796155:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.796156:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 15 buffers still posted 00000100:00000001:3.0:1773941498.796158:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.796159:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.796160:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.796162:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379108. 00000400:00000200:3.0:1773941498.796165:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18b98240 00000400:00000010:3.0:1773941498.796166:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18b98240. 00000100:00000001:3.0:1773941498.796169:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.796173:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.796179:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.796181:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 14 buffers still posted 00000100:00000001:3.0:1773941498.796182:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.796184:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.796186:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.796187:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379a50. 00000400:00000200:3.0:1773941498.796190:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18b98e40 00000400:00000010:3.0:1773941498.796192:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18b98e40. 00000100:00000001:3.0:1773941498.796195:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.796196:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.796201:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.796203:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 13 buffers still posted 00000100:00000001:3.0:1773941498.796205:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.796206:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.796208:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.796209:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379f78. 00000400:00000200:3.0:1773941498.796212:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18b98b40 00000400:00000010:3.0:1773941498.796213:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18b98b40. 00000100:00000001:3.0:1773941498.796228:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.796230:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.796235:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.796237:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 12 buffers still posted 00000100:00000001:3.0:1773941498.796239:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.796241:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.796242:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.796243:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379688. 00000400:00000200:3.0:1773941498.796246:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18b98cc0 00000400:00000010:3.0:1773941498.796248:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18b98cc0. 00000100:00000001:3.0:1773941498.796250:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.796251:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.796257:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.796259:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 11 buffers still posted 00000100:00000001:3.0:1773941498.796260:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.796261:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.796262:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.796264:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379478. 00000400:00000200:3.0:1773941498.796265:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d21c4cc00 00000400:00000010:3.0:1773941498.796297:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d21c4cc00. 00000100:00000001:3.0:1773941498.796301:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.796302:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.796305:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.796306:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 10 buffers still posted 00000100:00000001:3.0:1773941498.796307:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.796327:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.796328:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.796330:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379738. 00000400:00000200:3.0:1773941498.796333:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d21c4c900 00000400:00000010:3.0:1773941498.796335:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d21c4c900. 00000100:00000001:3.0:1773941498.796338:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.796339:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.796408:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.796410:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 9 buffers still posted 00000100:00000001:3.0:1773941498.796411:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.796412:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.796413:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.796415:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203790b0. 00000400:00000200:3.0:1773941498.796417:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d21c4c180 00000400:00000010:3.0:1773941498.796418:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d21c4c180. 00000100:00000001:3.0:1773941498.796419:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.796420:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.796423:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.796424:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 8 buffers still posted 00000100:00000001:3.0:1773941498.796425:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.796425:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.796426:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.796427:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379dc0. 00000400:00000200:3.0:1773941498.796428:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d21c4cf00 00000400:00000010:3.0:1773941498.796429:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d21c4cf00. 00000100:00000001:3.0:1773941498.796430:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.796431:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.796434:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.796435:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 7 buffers still posted 00000100:00000001:3.0:1773941498.796437:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.796437:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.796438:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.796443:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203793c8. 00000400:00000200:3.0:1773941498.796445:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d21c4ca80 00000400:00000010:3.0:1773941498.796446:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d21c4ca80. 00000100:00000001:3.0:1773941498.796448:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.796449:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.796452:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.796453:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 6 buffers still posted 00000100:00000001:3.0:1773941498.796454:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.796456:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.796457:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.796458:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203798f0. 00000400:00000200:3.0:1773941498.796460:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d21c4c840 00000400:00000010:3.0:1773941498.796461:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d21c4c840. 00000100:00000001:3.0:1773941498.796463:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.796463:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.796467:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.796468:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 5 buffers still posted 00000100:00000001:3.0:1773941498.796469:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.796471:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.796472:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.796473:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379d68. 00000400:00000200:3.0:1773941498.796474:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d21c4c3c0 00000400:00000010:3.0:1773941498.796475:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d21c4c3c0. 00000100:00000001:3.0:1773941498.796477:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.796478:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.796482:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.796483:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 4 buffers still posted 00000100:00000001:3.0:1773941498.796484:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.796485:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.796486:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.796487:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379f20. 00000400:00000200:3.0:1773941498.796489:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d21c4c780 00000400:00000010:3.0:1773941498.796491:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d21c4c780. 00000100:00000001:3.0:1773941498.796492:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.796493:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.796496:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.796498:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 3 buffers still posted 00000100:00000001:3.0:1773941498.796499:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.796504:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.796506:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.796507:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379b58. 00000400:00000200:3.0:1773941498.796508:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d21c4c000 00000400:00000010:3.0:1773941498.796509:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d21c4c000. 00000100:00000001:3.0:1773941498.796512:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.796512:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.796516:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.796517:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 2 buffers still posted 00000100:00000001:3.0:1773941498.796519:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.796520:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.796521:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.796523:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379cb8. 00000400:00000200:3.0:1773941498.796525:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d21c4c6c0 00000400:00000010:3.0:1773941498.796527:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d21c4c6c0. 00000100:00000001:3.0:1773941498.796529:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.796530:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.796535:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.796536:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 1 buffers still posted 00000100:00000001:3.0:1773941498.796538:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941498.796540:0:461:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941498.796541:0:461:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941498.796542:0:461:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379000. 00000400:00000200:3.0:1773941498.796545:0:461:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d21c4c300 00000400:00000010:3.0:1773941498.796546:0:461:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d21c4c300. 00000100:00000001:3.0:1773941498.796549:0:461:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941498.796549:0:461:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941498.796552:0:461:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941498.796553:0:461:0:(events.c:349:request_in_callback()) Buffer complete: 0 buffers still posted 00000100:00000001:3.0:1773941498.796554:0:461:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:3.0:1773941498.796556:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796557:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796559:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796561:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796562:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796563:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796565:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796565:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796570:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796571:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796572:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796573:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796575:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796576:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796578:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796580:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796582:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796584:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796585:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796587:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796589:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796591:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796594:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796596:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796598:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796600:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796602:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796604:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796607:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796608:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796609:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796611:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796613:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796615:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796616:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796618:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796620:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796621:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796623:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796624:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796627:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796628:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796631:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796632:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796633:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796634:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796636:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796637:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796644:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796645:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796646:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796647:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796650:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796651:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796652:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796653:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796654:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796655:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796656:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796656:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796658:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796658:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796659:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796660:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796661:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796662:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796663:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796664:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796665:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796666:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796667:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796668:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796669:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796670:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796671:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796671:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796673:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796673:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796674:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796675:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796676:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796677:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796678:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796679:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796680:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796680:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796682:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796682:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796686:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796687:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796688:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796689:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796691:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796692:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796693:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796694:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796695:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796696:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796696:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796697:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796698:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796699:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796699:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796700:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796701:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796702:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796703:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796704:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796706:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796706:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796707:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796708:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796708:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796709:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796710:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796711:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796711:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796712:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796713:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796715:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796715:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796716:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796717:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796718:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796719:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796719:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.796721:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.796722:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000010:3.0:1773941498.796727:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d18a1a000. 00000100:00000010:3.0:1773941498.796731:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1815a000. 00000100:00000010:3.0:1773941498.796733:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d18a1e000. 00000100:00000010:3.0:1773941498.796735:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d18158800. 00000100:00000010:3.0:1773941498.796739:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d07fd0000. 00000100:00000010:3.0:1773941498.796741:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1815bc00. 00000100:00000010:3.0:1773941498.796743:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d07fd4000. 00000100:00000010:3.0:1773941498.796746:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d18159000. 00000100:00000010:3.0:1773941498.796748:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d07fd6000. 00000100:00000010:3.0:1773941498.796751:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d18159800. 00000100:00000010:3.0:1773941498.796752:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d07fd2000. 00000100:00000010:3.0:1773941498.796754:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1815ac00. 00000100:00000010:3.0:1773941498.796755:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08a00000. 00000100:00000010:3.0:1773941498.796758:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d18158400. 00000100:00000010:3.0:1773941498.796760:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08a04000. 00000100:00000010:3.0:1773941498.796763:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1815a400. 00000100:00000010:3.0:1773941498.796764:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08a06000. 00000100:00000010:3.0:1773941498.796767:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1815b400. 00000100:00000010:3.0:1773941498.796768:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08a02000. 00000100:00000010:3.0:1773941498.796770:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d18158000. 00000100:00000010:3.0:1773941498.796771:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d111fc000. 00000100:00000010:3.0:1773941498.796773:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1815a800. 00000100:00000010:3.0:1773941498.796775:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d111fe000. 00000100:00000010:3.0:1773941498.796776:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d18158c00. 00000100:00000010:3.0:1773941498.796778:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d111fa000. 00000100:00000010:3.0:1773941498.796779:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1815b800. 00000100:00000010:3.0:1773941498.796781:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d111f8000. 00000100:00000010:3.0:1773941498.796782:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1815b000. 00000100:00000010:3.0:1773941498.796783:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08a1e000. 00000100:00000010:3.0:1773941498.796786:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d107d0000. 00000100:00000010:3.0:1773941498.796787:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08a1a000. 00000100:00000010:3.0:1773941498.796789:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d107d2800. 00000100:00000010:3.0:1773941498.796791:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08a18000. 00000100:00000010:3.0:1773941498.796796:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d107d0c00. 00000100:00000010:3.0:1773941498.796797:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08a1c000. 00000100:00000010:3.0:1773941498.796799:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d107d3800. 00000100:00000010:3.0:1773941498.796802:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08a10000. 00000100:00000010:3.0:1773941498.796814:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d107d3000. 00000100:00000010:3.0:1773941498.796815:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08a14000. 00000100:00000010:3.0:1773941498.796817:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d107d1c00. 00000100:00000010:3.0:1773941498.796820:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08a16000. 00000100:00000010:3.0:1773941498.796822:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d107d1400. 00000100:00000010:3.0:1773941498.796824:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08a12000. 00000100:00000010:3.0:1773941498.796826:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d107d2000. 00000100:00000010:3.0:1773941498.796828:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08a08000. 00000100:00000010:3.0:1773941498.796830:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d107d0800. 00000100:00000010:3.0:1773941498.796831:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08a0c000. 00000100:00000010:3.0:1773941498.796833:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d107d3c00. 00000100:00000010:3.0:1773941498.796834:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08a0e000. 00000100:00000010:3.0:1773941498.796836:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d107d1000. 00000100:00000010:3.0:1773941498.796837:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08a0a000. 00000100:00000010:3.0:1773941498.796839:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d107d1800. 00000100:00000010:3.0:1773941498.796840:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d03320000. 00000100:00000010:3.0:1773941498.796844:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d107d2c00. 00000100:00000010:3.0:1773941498.796846:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d03324000. 00000100:00000010:3.0:1773941498.796848:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d107d0400. 00000100:00000010:3.0:1773941498.796849:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d03326000. 00000100:00000010:3.0:1773941498.796851:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d107d2400. 00000100:00000010:3.0:1773941498.796853:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d03322000. 00000100:00000010:3.0:1773941498.796855:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d107d3400. 00000100:00000010:3.0:1773941498.796859:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d06224000. 00000100:00000010:3.0:1773941498.796879:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d19724400. 00000100:00000010:3.0:1773941498.796883:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d06226000. 00000100:00000010:3.0:1773941498.796886:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d19727800. 00000100:00000010:3.0:1773941498.796889:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d06222000. 00000100:00000010:3.0:1773941498.796893:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1125a400. 00000100:00000010:3.0:1773941498.796914:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d089e2000. 00000100:00000010:3.0:1773941498.796917:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1125ac00. 00000100:00000010:3.0:1773941498.796920:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d089e6000. 00000100:00000010:3.0:1773941498.796923:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1125a800. 00000100:00000010:3.0:1773941498.796925:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d089e4000. 00000100:00000010:3.0:1773941498.796927:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d11258000. 00000100:00000010:3.0:1773941498.796928:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d089e0000. 00000100:00000010:3.0:1773941498.796930:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1125b000. 00000100:00000010:3.0:1773941498.796932:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08350000. 00000100:00000010:3.0:1773941498.796934:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1125b800. 00000100:00000010:3.0:1773941498.796935:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08352000. 00000100:00000010:3.0:1773941498.796937:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d11259400. 00000100:00000010:3.0:1773941498.796938:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08356000. 00000100:00000010:3.0:1773941498.796940:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d11258c00. 00000100:00000010:3.0:1773941498.796942:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08354000. 00000100:00000010:3.0:1773941498.796943:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d02da7c00. 00000100:00000010:3.0:1773941498.796945:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d111be000. 00000100:00000010:3.0:1773941498.796954:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d0923f400. 00000100:00000010:3.0:1773941498.796956:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d111bc000. 00000100:00000010:3.0:1773941498.796958:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d0923d400. 00000100:00000010:3.0:1773941498.797095:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d111b8000. 00000100:00000010:3.0:1773941498.797099:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d0923d000. 00000100:00000010:3.0:1773941498.797101:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d111ba000. 00000100:00000010:3.0:1773941498.797103:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d0923e400. 00000100:00000010:3.0:1773941498.797105:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d089fa000. 00000100:00000010:3.0:1773941498.797107:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d0923e800. 00000100:00000010:3.0:1773941498.797109:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d089fe000. 00000100:00000010:3.0:1773941498.797110:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d0923ec00. 00000100:00000010:3.0:1773941498.797112:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d089fc000. 00000100:00000010:3.0:1773941498.797113:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d0923cc00. 00000100:00000010:3.0:1773941498.797115:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d089f8000. 00000100:00000010:3.0:1773941498.797117:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d0923f800. 00000100:00000010:3.0:1773941498.797118:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d18a32000. 00000100:00000010:3.0:1773941498.797125:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d0923c800. 00000100:00000010:3.0:1773941498.797126:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d18a36000. 00000100:00000010:3.0:1773941498.797128:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d0923d800. 00000100:00000010:3.0:1773941498.797129:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d18a34000. 00000100:00000010:3.0:1773941498.797131:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d0923c400. 00000100:00000010:3.0:1773941498.797133:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d18a30000. 00000100:00000010:3.0:1773941498.797134:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d0923fc00. 00000100:00000010:3.0:1773941498.797135:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d10814000. 00000100:00000010:3.0:1773941498.797149:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d0923c000. 00000100:00000010:3.0:1773941498.797150:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d10810000. 00000100:00000010:3.0:1773941498.797152:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d0923dc00. 00000100:00000010:3.0:1773941498.797153:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d10812000. 00000100:00000010:3.0:1773941498.797155:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d03074800. 00000100:00000010:3.0:1773941498.797156:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d10816000. 00000100:00000010:3.0:1773941498.797158:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d11273800. 00000100:00000010:3.0:1773941498.797160:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d03506000. 00000100:00000010:3.0:1773941498.797162:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d181d0400. 00000100:00000010:3.0:1773941498.797163:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d03504000. 00000100:00000010:3.0:1773941498.797165:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d181d1800. 00000100:00000010:3.0:1773941498.797167:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d03500000. 00000100:00000010:3.0:1773941498.797168:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d181d1400. 00000100:00000010:3.0:1773941498.797169:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d03502000. 00000100:00000010:3.0:1773941498.797171:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d181d0000. 00000100:00000010:3.0:1773941498.797173:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d0357e000. 00000100:00000010:3.0:1773941498.797175:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d181d2c00. 00000100:00000010:3.0:1773941498.797176:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d0357c000. 00000100:00000010:3.0:1773941498.797178:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d181d3400. 00000100:00000010:3.0:1773941498.797180:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d03578000. 00000100:00000010:3.0:1773941498.797181:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d181d2400. 00000100:00000010:3.0:1773941498.797239:0:461:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff988d08992000. 00000100:00000010:3.0:1773941498.797243:0:461:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff988d08994800. 00000100:00000010:3.0:1773941498.797245:0:461:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff988d19767800. 00000100:00000001:3.0:1773941498.797247:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797254:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797256:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797257:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797258:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797258:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797260:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797260:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797261:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797262:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797263:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797263:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797264:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797265:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797266:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797267:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797268:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797269:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797270:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797271:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797273:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797273:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797274:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797276:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797277:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797277:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797278:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797279:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797280:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797280:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797282:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797283:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797283:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797284:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797286:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797286:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797287:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797288:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797289:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797290:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797291:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797296:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797298:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797299:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797301:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797302:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797303:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797304:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797305:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797306:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797307:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797307:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797308:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797309:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797310:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797310:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797311:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797312:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797313:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797314:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797314:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797315:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797316:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797317:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797318:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797318:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797319:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797320:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797321:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797322:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797323:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797324:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797325:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797326:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797328:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797328:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797329:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797330:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797332:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797333:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797334:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797339:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797340:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797342:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797342:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797343:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797344:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797345:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797346:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797347:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797348:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797349:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797350:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797351:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797352:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797352:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797353:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797354:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797355:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797355:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797356:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797357:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797358:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797359:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797360:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797361:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797361:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797362:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797363:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797364:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797366:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797366:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797367:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797368:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797369:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797370:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797371:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797372:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797373:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797374:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797375:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797380:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797381:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797382:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797383:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797384:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941498.797385:0:461:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941498.797385:0:461:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000010:3.0:1773941498.797387:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d0357a000. 00000100:00000010:3.0:1773941498.797390:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d181d2800. 00000100:00000010:3.0:1773941498.797391:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d07138000. 00000100:00000010:3.0:1773941498.797406:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d181d3c00. 00000100:00000010:3.0:1773941498.797408:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d0713a000. 00000100:00000010:3.0:1773941498.797409:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d181d1000. 00000100:00000010:3.0:1773941498.797411:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d0713e000. 00000100:00000010:3.0:1773941498.797413:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d181d2000. 00000100:00000010:3.0:1773941498.797415:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d0713c000. 00000100:00000010:3.0:1773941498.797416:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d181d0800. 00000100:00000010:3.0:1773941498.797418:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08a2c000. 00000100:00000010:3.0:1773941498.797420:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d181d3800. 00000100:00000010:3.0:1773941498.797422:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08a28000. 00000100:00000010:3.0:1773941498.797424:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d181d3000. 00000100:00000010:3.0:1773941498.797426:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08a2a000. 00000100:00000010:3.0:1773941498.797427:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1a052000. 00000100:00000010:3.0:1773941498.797429:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08a2e000. 00000100:00000010:3.0:1773941498.797431:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1a92e000. 00000100:00000010:3.0:1773941498.797433:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d18a0e000. 00000100:00000010:3.0:1773941498.797435:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1a92c800. 00000100:00000010:3.0:1773941498.797437:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d18a0c000. 00000100:00000010:3.0:1773941498.797439:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1a92fc00. 00000100:00000010:3.0:1773941498.797441:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d18a08000. 00000100:00000010:3.0:1773941498.797442:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1a3b7c00. 00000100:00000010:3.0:1773941498.797444:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d18a0a000. 00000100:00000010:3.0:1773941498.797447:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d21caf000. 00000100:00000010:3.0:1773941498.797450:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08aec000. 00000100:00000010:3.0:1773941498.797467:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d21caec00. 00000100:00000010:3.0:1773941498.797470:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08ae8000. 00000100:00000010:3.0:1773941498.797472:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d21caf400. 00000100:00000010:3.0:1773941498.797474:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08aea000. 00000100:00000010:3.0:1773941498.797475:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d42041c00. 00000100:00000010:3.0:1773941498.797477:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08aee000. 00000100:00000010:3.0:1773941498.797479:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d42041400. 00000100:00000010:3.0:1773941498.797481:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d05420000. 00000100:00000010:3.0:1773941498.797483:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d42043c00. 00000100:00000010:3.0:1773941498.797486:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d05422000. 00000100:00000010:3.0:1773941498.797487:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d42041000. 00000100:00000010:3.0:1773941498.797489:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d05426000. 00000100:00000010:3.0:1773941498.797490:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d42040400. 00000100:00000010:3.0:1773941498.797492:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d05424000. 00000100:00000010:3.0:1773941498.797494:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d42042400. 00000100:00000010:3.0:1773941498.797495:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d065ba000. 00000100:00000010:3.0:1773941498.797497:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d180be000. 00000100:00000010:3.0:1773941498.797499:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d065be000. 00000100:00000010:3.0:1773941498.797502:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d180bd400. 00000100:00000010:3.0:1773941498.797503:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d065bc000. 00000100:00000010:3.0:1773941498.797506:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1a1b1400. 00000100:00000010:3.0:1773941498.797508:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d065b8000. 00000100:00000010:3.0:1773941498.797510:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1a1b1000. 00000100:00000010:3.0:1773941498.797511:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d20324000. 00000100:00000010:3.0:1773941498.797525:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1a1b1c00. 00000100:00000010:3.0:1773941498.797527:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d20320000. 00000100:00000010:3.0:1773941498.797528:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1a1b0800. 00000100:00000010:3.0:1773941498.797530:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d20322000. 00000100:00000010:3.0:1773941498.797532:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1a1b3c00. 00000100:00000010:3.0:1773941498.797533:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d20326000. 00000100:00000010:3.0:1773941498.797535:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d02ce7400. 00000100:00000010:3.0:1773941498.797537:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d06518000. 00000100:00000010:3.0:1773941498.797539:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d02ce7000. 00000100:00000010:3.0:1773941498.797545:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d0651a000. 00000100:00000010:3.0:1773941498.797547:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d02ce6000. 00000100:00000010:3.0:1773941498.797548:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d0651e000. 00000100:00000010:3.0:1773941498.797550:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d02ce5800. 00000100:00000010:3.0:1773941498.797552:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d0651c000. 00000100:00000010:3.0:1773941498.797554:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d02ce6800. 00000100:00000010:3.0:1773941498.797556:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d065a6000. 00000100:00000010:3.0:1773941498.797558:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d02ce4800. 00000100:00000010:3.0:1773941498.797559:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d065a4000. 00000100:00000010:3.0:1773941498.797561:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d02ce5000. 00000100:00000010:3.0:1773941498.797562:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d065a0000. 00000100:00000010:3.0:1773941498.797564:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d02ce7c00. 00000100:00000010:3.0:1773941498.797565:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d065a2000. 00000100:00000010:3.0:1773941498.797567:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d02ce4c00. 00000100:00000010:3.0:1773941498.797568:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d18904000. 00000100:00000010:3.0:1773941498.797584:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d02ce6400. 00000100:00000010:3.0:1773941498.797587:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d18900000. 00000100:00000010:3.0:1773941498.797590:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d02ce5c00. 00000100:00000010:3.0:1773941498.797593:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d18902000. 00000100:00000010:3.0:1773941498.797596:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d02ce7800. 00000100:00000010:3.0:1773941498.797598:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d18906000. 00000100:00000010:3.0:1773941498.797601:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d02ce6c00. 00000100:00000010:3.0:1773941498.797604:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d0650c000. 00000100:00000010:3.0:1773941498.797607:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d02ce5400. 00000100:00000010:3.0:1773941498.797610:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d06508000. 00000100:00000010:3.0:1773941498.797612:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d02ce4400. 00000100:00000010:3.0:1773941498.797615:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d0650a000. 00000100:00000010:3.0:1773941498.797618:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d203e7c00. 00000100:00000010:3.0:1773941498.797623:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d0650e000. 00000100:00000010:3.0:1773941498.797626:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d203e5000. 00000100:00000010:3.0:1773941498.797628:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d0897e000. 00000100:00000010:3.0:1773941498.797632:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d203e6400. 00000100:00000010:3.0:1773941498.797635:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d0897c000. 00000100:00000010:3.0:1773941498.797643:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d203e7400. 00000100:00000010:3.0:1773941498.797645:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08978000. 00000100:00000010:3.0:1773941498.797647:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d203e6c00. 00000100:00000010:3.0:1773941498.797649:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d0897a000. 00000100:00000010:3.0:1773941498.797651:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d203e6000. 00000100:00000010:3.0:1773941498.797652:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d089d4000. 00000100:00000010:3.0:1773941498.797667:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d203e4800. 00000100:00000010:3.0:1773941498.797669:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d089d0000. 00000100:00000010:3.0:1773941498.797670:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d203e7000. 00000100:00000010:3.0:1773941498.797672:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d089d2000. 00000100:00000010:3.0:1773941498.797674:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d203e4c00. 00000100:00000010:3.0:1773941498.797675:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d089d6000. 00000100:00000010:3.0:1773941498.797677:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d203e5800. 00000100:00000010:3.0:1773941498.797679:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d18914000. 00000100:00000010:3.0:1773941498.797681:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d035cec00. 00000100:00000010:3.0:1773941498.797682:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d18910000. 00000100:00000010:3.0:1773941498.797684:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d035cd800. 00000100:00000010:3.0:1773941498.797686:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d18912000. 00000100:00000010:3.0:1773941498.797688:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d035cd000. 00000100:00000010:3.0:1773941498.797689:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d18916000. 00000100:00000010:3.0:1773941498.797691:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d035cfc00. 00000100:00000010:3.0:1773941498.797693:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d02d16000. 00000100:00000010:3.0:1773941498.797695:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d035cc800. 00000100:00000010:3.0:1773941498.797697:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d02d14000. 00000100:00000010:3.0:1773941498.797699:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d035ce000. 00000100:00000010:3.0:1773941498.797700:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d02d10000. 00000100:00000010:3.0:1773941498.797702:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d035cd400. 00000100:00000010:3.0:1773941498.797704:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d02d12000. 00000100:00000010:3.0:1773941498.797706:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d035cdc00. 00000100:00000010:3.0:1773941498.797707:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d0352e000. 00000100:00000010:3.0:1773941498.797709:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d035cf000. 00000100:00000010:3.0:1773941498.797711:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d0352c000. 00000100:00000010:3.0:1773941498.797713:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d035cf800. 00000100:00000010:3.0:1773941498.797719:0:461:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d03528000. 00000100:00000010:3.0:1773941498.797727:0:461:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d035ccc00. 00000100:00000010:3.0:1773941498.797729:0:461:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff988d181c5000. 00000100:00000010:3.0:1773941498.797731:0:461:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff988d0835a800. 00000100:00000010:3.0:1773941498.797733:0:461:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff988d181c3000. 00000100:00000001:3.0:1773941498.797736:0:461:0:(nrs.c:1051:nrs_svcpt_cleanup_locked()) Process entered 00000100:00000001:3.0:1773941498.797738:0:461:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941498.797740:0:461:0:(nrs.c:111:nrs_policy_stop_locked()) Process entered 00000100:00000001:3.0:1773941498.797742:0:461:0:(nrs.c:67:nrs_policy_stop0()) Process entered 00000100:00000010:3.0:1773941498.797743:0:461:0:(nrs_fifo.c:83:nrs_fifo_stop()) kfreed 'head': 40 at ffff988d209c1a80. 00000100:00000001:3.0:1773941498.797746:0:461:0:(nrs.c:85:nrs_policy_stop0()) Process leaving 00000100:00000001:3.0:1773941498.797747:0:461:0:(nrs.c:147:nrs_policy_stop_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941498.797749:0:461:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941498.797750:0:461:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d035da300. 00000100:00000001:3.0:1773941498.797752:0:461:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941498.797753:0:461:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941498.797754:0:461:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d035dacc0. 00000100:00000001:3.0:1773941498.797755:0:461:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941498.797756:0:461:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941498.797757:0:461:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d035daa80. 00000100:00000001:3.0:1773941498.797758:0:461:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941498.797759:0:461:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941498.797760:0:461:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d035dad80. 00000100:00000001:3.0:1773941498.797762:0:461:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941498.797762:0:461:0:(nrs.c:111:nrs_policy_stop_locked()) Process entered 00000100:00000001:3.0:1773941498.797763:0:461:0:(nrs.c:67:nrs_policy_stop0()) Process entered 00000100:00000010:3.0:1773941498.797764:0:461:0:(nrs_fifo.c:83:nrs_fifo_stop()) kfreed 'head': 40 at ffff988d209c1cc0. 00000100:00000001:3.0:1773941498.797766:0:461:0:(nrs.c:85:nrs_policy_stop0()) Process leaving 00000100:00000001:3.0:1773941498.797767:0:461:0:(nrs.c:147:nrs_policy_stop_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941498.797768:0:461:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941498.797769:0:461:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d2030b6c0. 00000100:00000001:3.0:1773941498.797770:0:461:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941498.797771:0:461:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941498.797772:0:461:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d2030bcc0. 00000100:00000001:3.0:1773941498.797773:0:461:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941498.797774:0:461:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941498.797775:0:461:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d2030b900. 00000100:00000001:3.0:1773941498.797776:0:461:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941498.797777:0:461:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941498.797783:0:461:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d2030be40. 00000100:00000010:3.0:1773941498.797785:0:461:0:(nrs.c:1080:nrs_svcpt_cleanup_locked()) kfreed 'nrs': 112 at ffff988d0716b700. 00000100:00000001:3.0:1773941498.797786:0:461:0:(nrs.c:1082:nrs_svcpt_cleanup_locked()) Process leaving 00000100:00000001:3.0:1773941498.797787:0:461:0:(nrs.c:1051:nrs_svcpt_cleanup_locked()) Process entered 00000100:00000001:3.0:1773941498.797788:0:461:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941498.797788:0:461:0:(nrs.c:111:nrs_policy_stop_locked()) Process entered 00000100:00000001:3.0:1773941498.797789:0:461:0:(nrs.c:67:nrs_policy_stop0()) Process entered 00000100:00000010:3.0:1773941498.797790:0:461:0:(nrs_fifo.c:83:nrs_fifo_stop()) kfreed 'head': 40 at ffff988d209c1b00. 00000100:00000001:3.0:1773941498.797792:0:461:0:(nrs.c:85:nrs_policy_stop0()) Process leaving 00000100:00000001:3.0:1773941498.797793:0:461:0:(nrs.c:147:nrs_policy_stop_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941498.797794:0:461:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941498.797795:0:461:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d2030b0c0. 00000100:00000001:3.0:1773941498.797796:0:461:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941498.797797:0:461:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941498.797798:0:461:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d2030b180. 00000100:00000001:3.0:1773941498.797799:0:461:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941498.797800:0:461:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941498.797801:0:461:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d2030b540. 00000100:00000001:3.0:1773941498.797802:0:461:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941498.797803:0:461:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941498.797804:0:461:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d2030b9c0. 00000100:00000001:3.0:1773941498.797805:0:461:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941498.797806:0:461:0:(nrs.c:111:nrs_policy_stop_locked()) Process entered 00000100:00000001:3.0:1773941498.797806:0:461:0:(nrs.c:67:nrs_policy_stop0()) Process entered 00000100:00000010:3.0:1773941498.797807:0:461:0:(nrs_fifo.c:83:nrs_fifo_stop()) kfreed 'head': 40 at ffff988d209c1140. 00000100:00000001:3.0:1773941498.797809:0:461:0:(nrs.c:85:nrs_policy_stop0()) Process leaving 00000100:00000001:3.0:1773941498.797810:0:461:0:(nrs.c:147:nrs_policy_stop_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941498.797811:0:461:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941498.797812:0:461:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d2030b600. 00000100:00000001:3.0:1773941498.797813:0:461:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941498.797813:0:461:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941498.797814:0:461:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d2030bf00. 00000100:00000001:3.0:1773941498.797816:0:461:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941498.797816:0:461:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941498.797818:0:461:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d2030b840. 00000100:00000001:3.0:1773941498.797819:0:461:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941498.797819:0:461:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941498.797820:0:461:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d2030b3c0. 00000100:00000010:3.0:1773941498.797821:0:461:0:(nrs.c:1080:nrs_svcpt_cleanup_locked()) kfreed 'nrs': 112 at ffff988d0716be00. 00000100:00000001:3.0:1773941498.797823:0:461:0:(nrs.c:1082:nrs_svcpt_cleanup_locked()) Process leaving 00000020:00000010:3.0:1773941498.797905:0:461:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 3680 at ffff988d0834f000. 00000020:00000010:3.0:1773941498.797913:0:461:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff988d035ce800. 00000100:00000010:3.0:1773941498.797939:0:461:0:(service.c:3818:ptlrpc_service_free()) kfreed 'array->paa_reqs_array': 12080 at ffff988d08950000. 00000100:00000010:3.0:1773941498.797947:0:461:0:(service.c:3824:ptlrpc_service_free()) kfreed 'array->paa_reqs_count': 3020 at ffff988d08349000. 00000100:00000010:3.0:1773941498.797949:0:461:0:(service.c:3818:ptlrpc_service_free()) kfreed 'array->paa_reqs_array': 12080 at ffff988d089c0000. 00000100:00000010:3.0:1773941498.797955:0:461:0:(service.c:3824:ptlrpc_service_free()) kfreed 'array->paa_reqs_count': 3020 at ffff988d0834e000. 00000100:00000010:3.0:1773941498.797957:0:461:0:(service.c:3830:ptlrpc_service_free()) kfreed 'svcpt': 784 at ffff988d18159400. 00000100:00000010:3.0:1773941498.797959:0:461:0:(service.c:3830:ptlrpc_service_free()) kfreed 'svcpt': 784 at ffff988d181d0c00. 00000100:00000010:3.0:1773941498.797987:0:461:0:(service.c:3836:ptlrpc_service_free()) kfreed 'svc': 360 at ffff988d19735000. 00000100:00000001:3.0:1773941498.797990:0:461:0:(service.c:3863:ptlrpc_unregister_service()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941498.798070:0:14590:0:(ldlm_lockd.c:142:have_expired_locks()) Process entered 00010000:00000001:1.0:1773941498.798074:0:14590:0:(ldlm_lockd.c:147:have_expired_locks()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941498.798087:0:14590:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff988d03e7c580. 00000020:00000010:1.0:1773941498.798093:0:14590:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d0335f200. 00010000:00000010:3.0:1773941498.798096:0:461:0:(ldlm_lockd.c:3586:ldlm_cleanup()) kfreed 'ldlm_state': 32 at ffff988d0300e900. 00000040:00000010:1.0:1773941498.798097:0:14590:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d0335f600. 00010000:00000001:3.0:1773941498.798102:0:461:0:(ldlm_lockd.c:3589:ldlm_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941498.798103:0:14590:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d18b1c000. 00010000:00000001:3.0:1773941498.798106:0:461:0:(ldlm_lockd.c:3058:ldlm_put_ref()) Process leaving 00000020:00000010:1.0:1773941498.798108:0:14590:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff988d181f3250. 00010000:00000010:3.0:1773941498.798110:0:461:0:(ldlm_lib.c:647:client_obd_cleanup()) kfreed 'cli->cl_read_io_latency_by_size': 3920 at ffff988d035d6000. 00000020:00000010:1.0:1773941498.798113:0:14590:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d0335e400. 00010000:00000010:3.0:1773941498.798115:0:461:0:(ldlm_lib.c:651:client_obd_cleanup()) kfreed 'cli->cl_write_io_latency_by_size': 3920 at ffff988d035d1000. 80000000:00000010:1.0:1773941498.798118:0:14590:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d054f3c00. 00010000:00000001:3.0:1773941498.798120:0:461:0:(ldlm_lib.c:654:client_obd_cleanup()) Process leaving 40000000:00000010:1.0:1773941498.798123:0:14590:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff988d03e7c380. 10000000:00000010:3.0:1773941498.798124:0:461:0:(mgc_request.c:834:mgc_device_free()) kfreed 'lu': 64 at ffff988d209c17c0. 00100000:00000010:1.0:1773941498.798129:0:14590:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff988d18b1b000. 10000000:00000001:3.0:1773941498.798130:0:461:0:(mgc_request.c:836:mgc_device_free()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941498.798136:0:461:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d0381e000. 00040000:00000010:1.0:1773941498.798138:0:14590:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d02cfd000. 00000040:00000010:3.0:1773941498.798140:0:461:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d0381e400. 00000020:00000010:3.0:1773941498.798145:0:461:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d111b0000. 00040000:00000010:1.0:1773941498.798145:0:14590:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff988d18b1c800. 00000020:00000010:3.0:1773941498.798151:0:461:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d0381fa00. 00040000:00000010:1.0:1773941498.798154:0:14590:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d0335f400. 00040000:00000010:3.0:1773941498.798156:0:461:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d04c1a400. 00040000:00000010:3.0:1773941498.798162:0:461:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d0381fe00. 00000004:00000010:1.0:1773941498.798162:0:14590:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff988d18b1d800. 00000020:00000010:3.0:1773941498.798168:0:461:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0381ec00. 00000004:00000010:1.0:1773941498.798171:0:14590:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff988d065f6000. 00000020:00000001:3.0:1773941498.798173:0:461:0:(obd_class.h:647:obd_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941498.798178:0:461:0:(genops.c:66:obd_device_free()) slab-freed '(obd)': 6584 at ffff988d083919b8. 00000004:00000010:1.0:1773941498.798180:0:14590:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff988d18b1e800. 00000020:00000010:3.0:1773941498.798184:0:461:0:(genops.c:1220:obd_zombie_import_free()) kfreed 'imp': 1712 at ffff988d035c4800. 00002000:00000010:1.0:1773941498.798186:0:14590:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff988d1118b400. 00000020:00000001:3.0:1773941498.798189:0:461:0:(genops.c:1221:obd_zombie_import_free()) Process leaving 00000004:00000010:1.0:1773941498.798200:0:14590:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff988d1118a400. 00000020:00000010:1.0:1773941498.798203:0:14590:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0335e000. 00010000:00000001:1.0:1773941498.798206:0:14590:0:(ldlm_lockd.c:290:expired_lock_main()) Process leaving (rc=0 : 0 : 0) 00000080:00000010:0.0:1773941498.843495:0:14694:0:(super25.c:253:lustre_init_fs_context()) kmalloced '(lmd)': 128 at ffff988d181fe800. 00000020:00000001:0.0:1773941498.843503:0:14694:0:(obd_mount.c:1366:lustre_parse_monolithic()) Process entered 00000020:00000010:0.0:1773941498.843512:0:14694:0:(obd_mount.c:1388:lustre_parse_monolithic()) kmalloced '(lmd->lmd_params)': 4096 at ffff988d18a25000. 00000020:00000010:0.0:1773941498.843519:0:14694:0:(obd_mount.c:1096:lmd_parse_string()) kmalloced '(*handle)': 19 at ffff988d08957120. 00000020:00000010:0.0:1773941498.843523:0:14694:0:(obd_mount.c:1590:lustre_parse_monolithic()) kmalloced '(lmd->lmd_dev)': 48 at ffff988d108727c0. 00000020:00000010:0.0:1773941498.843527:0:14694:0:(obd_mount.c:1646:lustre_parse_monolithic()) kmalloced '(lmd->lmd_profile)': 14 at ffff988d065cd330. 00000020:01000004:0.0:1773941498.843529:0:14694:0:(obd_mount.c:937:lmd_print()) mount data: 00000020:01000004:0.0:1773941498.843530:0:14694:0:(obd_mount.c:939:lmd_print()) profile: lustre-client 00000020:01000004:0.0:1773941498.843531:0:14694:0:(obd_mount.c:940:lmd_print()) device: 192.168.204.114@tcp,192.168.204.124@tcp:/lustre 00000020:00000001:0.0:1773941498.843533:0:14694:0:(obd_mount.c:1686:lustre_parse_monolithic()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.843617:0:14694:0:(super25.c:131:lustre_fill_super()) Process entered 00000080:01200004:0.0:1773941498.843618:0:14694:0:(super25.c:132:lustre_fill_super()) VFS Op: sb ffff988d111b1800 00000020:00000001:0.0:1773941498.843620:0:14694:0:(obd_mount.c:652:lustre_init_lsi()) Process entered 00000020:00000010:0.0:1773941498.843622:0:14694:0:(obd_mount.c:656:lustre_init_lsi()) kmalloced '(lsi)': 1880 at ffff988d111b0000. 00000020:00000001:0.0:1773941498.843624:0:14694:0:(obd_mount.c:673:lustre_init_lsi()) Process leaving (rc=18446630330377633792 : -113743331917824 : ffff988d111b0000) 00000080:01000004:0.0:1773941498.843628:0:14694:0:(super25.c:169:lustre_fill_super()) Mounting client lustre-client 00000020:00000001:0.0:1773941498.843630:0:14694:0:(obd_mount.c:302:lustre_start_mgc()) Process entered 00000020:00000010:0.0:1773941498.843640:0:14694:0:(obd_mount.c:345:lustre_start_mgc()) kmalloced '(mgcname)': 23 at ffff988d06545ee0. 00000020:00000010:0.0:1773941498.843642:0:14694:0:(obd_mount.c:353:lustre_start_mgc()) kmalloced '(data)': 192 at ffff988d04fa2c00. 00000020:01000004:0.0:1773941498.843645:0:14694:0:(obd_mount.c:423:lustre_start_mgc()) Start MGC 'MGC192.168.204.114@tcp' 00000020:00000001:0.0:1773941498.843648:0:14694:0:(obd_mount.c:150:do_lcfg()) lcfg MGC192.168.204.114@tcp 0xcf005 192.168.204.114@tcp (null) (null) (null) 00000020:00000010:0.0:1773941498.843651:0:14694:0:(obd_mount.c:162:do_lcfg()) kmalloced '(lcfg)': 88 at ffff988d06580360. 00000020:00000080:0.0:1773941498.843653:0:14694:0:(obd_config.c:1506:class_process_config()) processing cmd: cf005 00000020:00000080:0.0:1773941498.843658:0:14694:0:(obd_config.c:1518:class_process_config()) adding mapping from uuid 192.168.204.114@tcp to nid 0x20000c0a8cc72 (192.168.204.114@tcp) 00000100:00000010:0.0:1773941498.843661:0:14694:0:(lustre_peer.c:71:class_add_uuid()) kmalloced '(data)': 704 at ffff988d065f8800. 00000100:00000040:0.0:1773941498.843665:0:14694:0:(lustre_peer.c:111:class_add_uuid()) found uuid 192.168.204.114@tcp 192.168.204.114@tcp cnt=1 00000400:00000200:0.0:1773941498.843668:0:14694:0:(peer.c:1295:LNetAddPeer()) Add peer with 1 NIDs 00000400:00001000:0.0:1773941498.843670:0:14694:0:(api-ni.c:3210:LNetNIInit()) refs 1 00000400:00000200:0.0:1773941498.843673:0:14694:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.204.114@tcp 00000400:00000200:0.0:1773941498.843677:0:14694:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941498.843681:0:14694:0:(peer.c:1737:lnet_peer_add()) peer 192.168.204.114@tcp NID flags 0x100001: -17 00000400:00000200:0.0:1773941498.843685:0:14694:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941498.843689:0:14694:0:(peer.c:2767:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:0.0:1773941498.843691:0:14694:0:(peer.c:2776:lnet_discover_peer_locked()) peer 192.168.204.114@tcp NID 192.168.204.114@tcp: 0. pending discovery 00000400:00000200:0.0:1773941498.843693:0:14694:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.204.114@tcp 00000100:00000010:0.0:1773941498.843695:0:14694:0:(lustre_peer.c:113:class_add_uuid()) kfreed 'data': 704 at ffff988d065f8800. 00000020:00000001:0.0:1773941498.843702:0:14694:0:(obd_config.c:1533:class_process_config()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1773941498.843704:0:14694:0:(obd_mount.c:168:do_lcfg()) kfreed 'lcfg': 88 at ffff988d06580360. 00000020:00000001:0.0:1773941498.843706:0:14694:0:(obd_mount.c:150:do_lcfg()) lcfg MGC192.168.204.114@tcp 0xcf005 192.168.204.114@tcp (null) (null) (null) 00000020:00000010:0.0:1773941498.843708:0:14694:0:(obd_mount.c:162:do_lcfg()) kmalloced '(lcfg)': 88 at ffff988d06580360. 00000020:00000080:0.0:1773941498.843709:0:14694:0:(obd_config.c:1506:class_process_config()) processing cmd: cf005 00000020:00000080:0.0:1773941498.843711:0:14694:0:(obd_config.c:1518:class_process_config()) adding mapping from uuid 192.168.204.114@tcp to nid 0x20000c0a8cc7c (192.168.204.124@tcp) 00000100:00000010:0.0:1773941498.843713:0:14694:0:(lustre_peer.c:71:class_add_uuid()) kmalloced '(data)': 704 at ffff988d065f8800. 00000100:00000040:0.0:1773941498.843715:0:14694:0:(lustre_peer.c:111:class_add_uuid()) found uuid 192.168.204.114@tcp 192.168.204.124@tcp cnt=2 00000400:00000200:0.0:1773941498.843716:0:14694:0:(peer.c:1295:LNetAddPeer()) Add peer with 2 NIDs 00000400:00001000:0.0:1773941498.843717:0:14694:0:(api-ni.c:3210:LNetNIInit()) refs 1 00000400:00000200:0.0:1773941498.843718:0:14694:0:(peer.c:1309:LNetAddPeer()) Adding NID[0]: 192.168.204.114@tcp 00000400:00000200:0.0:1773941498.843720:0:14694:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941498.843721:0:14694:0:(peer.c:1737:lnet_peer_add()) peer 192.168.204.114@tcp NID flags 0x100001: -17 00000400:00000200:0.0:1773941498.843723:0:14694:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941498.843725:0:14694:0:(peer.c:2767:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:0.0:1773941498.843726:0:14694:0:(peer.c:2776:lnet_discover_peer_locked()) peer 192.168.204.114@tcp NID 192.168.204.114@tcp: 0. pending discovery 00000400:00000200:0.0:1773941498.843727:0:14694:0:(peer.c:1360:LNetAddPeer()) A peer exists for NID 192.168.204.114@tcp 00000400:00000200:0.0:1773941498.843728:0:14694:0:(peer.c:1309:LNetAddPeer()) Adding NID[1]: 192.168.204.124@tcp 00000400:00000200:0.0:1773941498.843730:0:14694:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000010:0.0:1773941498.843733:0:14694:0:(peer.c:229:lnet_peer_alloc()) alloc '(lp)': 376 at ffff988d03357600. 00000400:00000200:0.0:1773941498.843735:0:14694:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941498.843736:0:14694:0:(peer.c:262:lnet_peer_alloc()) ffff988d03357600 nid 192.168.204.124@tcp 00000400:00000010:0.0:1773941498.843739:0:14694:0:(peer.c:193:lnet_peer_net_alloc()) alloc '(lpn)': 72 at ffff988d065802a0. 00000400:00000200:0.0:1773941498.843740:0:14694:0:(peer.c:202:lnet_peer_net_alloc()) ffff988d065802a0 net tcp 00000400:00000200:0.0:1773941498.843742:0:14694:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000010:0.0:1773941498.843743:0:14694:0:(peer.c:137:lnet_peer_ni_alloc()) alloc '(lpni)': 384 at ffff988d03356e00. 00000400:00000200:0.0:1773941498.843746:0:14694:0:(peer.c:183:lnet_peer_ni_alloc()) ffff988d03356e00 nid 192.168.204.124@tcp 00000400:00000200:0.0:1773941498.843751:0:14694:0:(peer.c:1091:lnet_peer_ni_clr_non_mr_pref_nid()) peer 192.168.204.124@tcp: -2 00000400:00000200:0.0:1773941498.843754:0:14694:0:(peer.c:1639:lnet_peer_attach_peer_ni()) peer 192.168.204.124@tcp NID 192.168.204.124@tcp flags 0x1 00000400:00000200:0.0:1773941498.843756:0:14694:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941498.843757:0:14694:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941498.843772:0:14694:0:(peer.c:2405:lnet_peer_queue_for_discovery()) Queue peer 192.168.204.124@tcp: 0 00000400:00000200:0.0:1773941498.843773:0:14694:0:(peer.c:2727:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:0.0:1773941498.843775:0:14694:0:(peer.c:2767:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:0.0:1773941498.843780:0:14694:0:(peer.c:2776:lnet_discover_peer_locked()) peer 192.168.204.124@tcp NID 192.168.204.124@tcp: 0. pending discovery 00000100:00000010:0.0:1773941498.843782:0:14694:0:(lustre_peer.c:113:class_add_uuid()) kfreed 'data': 704 at ffff988d065f8800. 00000020:00000001:0.0:1773941498.843784:0:14694:0:(obd_config.c:1533:class_process_config()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1773941498.843786:0:14694:0:(obd_mount.c:168:do_lcfg()) kfreed 'lcfg': 88 at ffff988d06580360. 00000020:00000010:0.0:1773941498.843789:0:14694:0:(obd_mount.c:487:lustre_start_mgc()) kmalloced '(uuid)': 40 at ffff988d10872d40. 00000020:01000004:0.0:1773941498.843793:0:14694:0:(obd_mount.c:208:lustre_start_simple()) Starting OBD MGC192.168.204.114@tcp (typ=mgc) 00000020:00000001:0.0:1773941498.843795:0:14694:0:(obd_mount.c:150:do_lcfg()) lcfg MGC192.168.204.114@tcp 0xcf001 mgc c1c1ff21-5f68-4bf6-bdc4-258644b24465 (null) (null) 00000020:00000010:0.0:1773941498.843798:0:14694:0:(obd_mount.c:162:do_lcfg()) kmalloced '(lcfg)': 120 at ffff988d181fe500. 00000400:00000200:3.0:1773941498.843800:0:2399:0:(peer.c:3988:lnet_peer_discovery_wait_for_work()) woken: 0 00000020:00000080:0.0:1773941498.843800:0:14694:0:(obd_config.c:1506:class_process_config()) processing cmd: cf001 00000020:00000001:0.0:1773941498.843805:0:14694:0:(obd_config.c:700:class_attach()) Process entered 00000020:00000001:0.0:1773941498.843808:0:14694:0:(genops.c:349:class_newdev()) Process entered 00000400:00000200:3.0:1773941498.843809:0:2399:0:(peer.c:4100:lnet_peer_discovery()) peer 192.168.204.124@tcp(ffff988d03357600) state 0x41 00000020:00000010:0.0:1773941498.843816:0:14694:0:(genops.c:51:obd_device_alloc()) slab-alloced 'obd': 6584 at ffff988d2039cd28. 00000400:00000010:3.0:1773941498.843817:0:2399:0:(api-ni.c:1821:lnet_ping_buffer_alloc()) alloc '(pbuf)': 281 at ffff988d1817d600. 00000020:00000080:0.0:1773941498.843823:0:14694:0:(genops.c:412:class_newdev()) Allocate new device MGC192.168.204.114@tcp (ffff988d2039cd28) 00000400:00000010:3.0:1773941498.843824:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac990. 00000020:00000001:0.0:1773941498.843826:0:14694:0:(genops.c:950:__class_new_export()) Process entered 00000020:00000010:0.0:1773941498.843831:0:14694:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff988d111b5800. 00000400:00000010:3.0:1773941498.843833:0:2399:0:(lib-lnet.h:478:lnet_rspt_alloc()) alloc 'rspt': 56 at ffff988d09a24c78. 00000020:00000001:0.0:1773941498.843834:0:14694:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:0.0:1773941498.843837:0:14694:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d111b5800 with handle 0x8e86767622e28cd to hash 00000020:00000001:0.0:1773941498.843838:0:14694:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00000020:00000001:0.0:1773941498.843841:0:14694:0:(obd_class.h:1008:obd_init_export()) Process entered 00000020:00000001:0.0:1773941498.843844:0:14694:0:(obd_class.h:1012:obd_init_export()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941498.843846:0:2399:0:(lib-move.c:5547:LNetGet()) LNetGet -> 12345-192.168.204.124@tcp 00000020:00000001:0.0:1773941498.843848:0:14694:0:(genops.c:1020:__class_new_export()) Process leaving (rc=18446630330377656320 : -113743331895296 : ffff988d111b5800) 00000020:00000040:0.0:1773941498.843851:0:14694:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b5800 : new refcount 1 00000020:00000040:0.0:1773941498.843855:0:14694:0:(obd_config.c:1030:class_incref()) incref MGC192.168.204.114@tcp (ffff988d2039cd28) now 2 - obd_device_list 00000400:00000200:3.0:1773941498.843857:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000020:00000001:0.0:1773941498.843860:0:14694:0:(genops.c:554:class_register_device()) Process leaving (rc=0 : 0 : 0) 00000020:00000080:0.0:1773941498.843863:0:14694:0:(obd_config.c:757:class_attach()) OBD: dev 0 attached type mgc with refcount 2 00000400:00000200:3.0:1773941498.843866:0:2399:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.124@tcp local destination 00000020:00000001:0.0:1773941498.843867:0:14694:0:(obd_config.c:759:class_attach()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941498.843870:0:14694:0:(obd_config.c:1512:class_process_config()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:3.0:1773941498.843872:0:2399:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000010:0.0:1773941498.843873:0:14694:0:(obd_mount.c:168:do_lcfg()) kfreed 'lcfg': 120 at ffff988d181fe500. 00000020:00000001:0.0:1773941498.843877:0:14694:0:(obd_mount.c:150:do_lcfg()) lcfg MGC192.168.204.114@tcp 0xcf003 MGS 192.168.204.114@tcp (null) (null) 00000400:00000200:3.0:1773941498.843878:0:2399:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000020:00000010:0.0:1773941498.843882:0:14694:0:(obd_mount.c:162:do_lcfg()) kmalloced '(lcfg)': 104 at ffff988d181fe500. 00000400:00000200:3.0:1773941498.843884:0:2399:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000020:00000080:0.0:1773941498.843886:0:14694:0:(obd_config.c:1506:class_process_config()) processing cmd: cf003 00000400:00000200:3.0:1773941498.843889:0:2399:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000020:00000001:0.0:1773941498.843890:0:14694:0:(obd_config.c:779:class_setup()) Process entered 00000400:00000200:3.0:1773941498.843894:0:2399:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [111:111:256:4294967295] 192.168.204.124@tcp LPNI seq info [1:1:8:4294967295] 00000400:00000200:3.0:1773941498.843898:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:3.0:1773941498.843903:0:2399:0:(lib-move.c:2009:lnet_handle_send()) rspt_next_hop_nid = 192.168.204.124@tcp 00000001:00000001:0.0:1773941498.843904:0:14694:0:(hash.c:1038:cfs_hash_create()) Process entered 00000001:00000010:0.0:1773941498.843909:0:14694:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 232 at ffff988d04c1a300. 00000400:00000200:3.0:1773941498.843910:0:2399:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.124@tcp:192.168.204.124@tcp) : GET try# 0 00000800:00000200:3.0:1773941498.843916:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 0 bytes in 0 frags to 12345-192.168.204.124@tcp 00000800:00000010:3.0:1773941498.843922:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 232 at ffff988d1a2ec800. 00000001:00000010:0.0:1773941498.843922:0:14694:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 32 at ffff988d06545400. 00000001:00000010:0.0:1773941498.843928:0:14694:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff988d03357000. 00000400:00000200:3.0:1773941498.843932:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000001:00000010:0.0:1773941498.843932:0:14694:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff988d03357a00. 00000001:00000010:0.0:1773941498.843937:0:14694:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff988d03356c00. 00000800:00000010:3.0:1773941498.843939:0:2399:0:(socklnd.c:145:ksocknal_create_peer()) alloc '(peer_ni)': 184 at ffff988d1837d6c0. 00000001:00000010:0.0:1773941498.843942:0:14694:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 344 at ffff988d03356a00. 00000800:00000010:3.0:1773941498.843945:0:2399:0:(socklnd.c:106:ksocknal_create_conn_cb()) alloc '(conn_cb)': 192 at ffff988d1837d780. 00000020:00000001:0.0:1773941498.843946:0:14694:0:(obd_class.h:557:obd_setup()) Process entered 00000800:00000200:3.0:1773941498.843952:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (2) 00000020:00000010:0.0:1773941498.843953:0:14694:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d03356200. 00000800:00000200:3.1:1773941498.843977:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (2) 00000020:00000010:0.0:1773941498.844130:0:14694:0:(lu_ucred.c:27:lu_ucred_key_init()) kmalloced '(value)': 128 at ffff988d181fef80. 00000020:00000010:0.0:1773941498.844140:0:14694:0:(tgt_main.c:756:tgt_ses_key_init()) slab-alloced 'session': 200 at ffff988d06522258. 00008000:00000010:0.0:1773941498.844146:0:14694:0:(echo_client.c:438:echo_session_key_init()) slab-alloced 'session': 8 at ffff988d065239c8. 00000020:00000010:0.0:1773941498.844153:0:14694:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d03356000. 00000020:00000010:0.0:1773941498.844157:0:14694:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d03357800. 00000040:00000010:0.0:1773941498.844161:0:14694:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d03357e00. 00000020:00000010:0.0:1773941498.844165:0:14694:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d111b0800. 00000020:00000010:0.0:1773941498.844170:0:14694:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d03356600. 00040000:00000010:0.0:1773941498.844175:0:14694:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d04c1a400. 00040000:00000010:0.0:1773941498.844180:0:14694:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d03356400. 10000000:00000001:0.0:1773941498.844184:0:14694:0:(mgc_request.c:850:mgc_device_alloc()) Process entered 10000000:00000010:0.0:1773941498.844187:0:14694:0:(mgc_request.c:852:mgc_device_alloc()) kmalloced '(lu)': 64 at ffff988d10872b40. 00000100:00000001:0.0:1773941498.844191:0:14694:0:(ptlrpcd.c:908:ptlrpcd_addref()) Process entered 00000100:00000001:0.0:1773941498.844194:0:14694:0:(ptlrpcd.c:917:ptlrpcd_addref()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941498.844198:0:14694:0:(ldlm_lib.c:298:client_obd_setup()) Process entered 00010000:00000010:0.0:1773941498.844206:0:14694:0:(ldlm_lib.c:421:client_obd_setup()) kmalloced '(cli->cl_read_io_latency_by_size)': 3920 at ffff988d18a20000. 00010000:00000010:0.0:1773941498.844230:0:14694:0:(ldlm_lib.c:434:client_obd_setup()) kmalloced '(cli->cl_write_io_latency_by_size)': 3920 at ffff988d18a26000. 00010000:00000001:0.0:1773941498.844242:0:14694:0:(ldlm_lockd.c:3030:ldlm_get_ref()) Process entered 00010000:00000001:0.0:1773941498.844244:0:14694:0:(ldlm_lockd.c:3351:ldlm_setup()) Process entered 00010000:00000010:0.0:1773941498.844247:0:14694:0:(ldlm_lockd.c:3356:ldlm_setup()) kmalloced '(ldlm_state)': 32 at ffff988d065457a0. 00010000:00000001:0.0:1773941498.844285:0:14694:0:(ldlm_resource.c:68:ldlm_debugfs_setup()) Process entered 00000400:00000200:3.0:1773941498.844294:0:2399:0:(peer.c:3788:lnet_peer_send_ping()) peer 192.168.204.124@tcp 00000400:00000200:3.0:1773941498.844298:0:2399:0:(peer.c:4118:lnet_peer_discovery()) peer 192.168.204.124@tcp(ffff988d03357600) state 0x241 rc 0 00010000:00000001:0.0:1773941498.844329:0:14694:0:(ldlm_resource.c:78:ldlm_debugfs_setup()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.844335:0:14694:0:(service.c:701:ptlrpc_register_service()) Process entered 00000100:00000010:0.0:1773941498.844342:0:14694:0:(service.c:745:ptlrpc_register_service()) kmalloced '(service)': 360 at ffff988d11267000. 00000100:00000010:0.0:1773941498.844347:0:14694:0:(service.c:803:ptlrpc_register_service()) kmalloced '(svcpt)': 784 at ffff988d065fa000. 00000100:00000010:0.0:1773941498.844364:0:14694:0:(service.c:627:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_array)': 12080 at ffff988d0653c000. 00000100:00000010:0.0:1773941498.844373:0:14694:0:(service.c:635:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_count)': 3020 at ffff988d18a24000. 00000100:00000010:0.0:1773941498.844385:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d065f9c00. 00000100:00000010:0.0:1773941498.844391:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d10806000. 00000100:00100000:0.0:1773941498.844394:0:14694:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/1 left), rc = 0 00000100:00000010:0.0:1773941498.844398:0:14694:0:(service.c:803:ptlrpc_register_service()) kmalloced '(svcpt)': 784 at ffff988d065fb000. 00000100:00000010:0.0:1773941498.844410:0:14694:0:(service.c:627:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_array)': 12080 at ffff988d18b48000. 00000100:00000010:0.0:1773941498.844418:0:14694:0:(service.c:635:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_count)': 3020 at ffff988d18a27000. 00000100:00000010:0.0:1773941498.844423:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d065f8000. 00000100:00000010:0.0:1773941498.844427:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d10802000. 00000100:00100000:0.0:1773941498.844429:0:14694:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_cbd: allocate 1 new 8192-byte reqbufs (0/1 left), rc = 0 00000400:00000200:0.0:1773941498.844434:0:14694:0:(lib-ptl.c:921:LNetSetLazyPortal()) Setting portal 15 lazy 00000020:00000010:0.0:1773941498.844472:0:14694:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff988d065f8c00. 00000020:00000010:0.0:1773941498.844477:0:14694:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 3680 at ffff988d18a21000. 00000100:00000001:0.0:1773941498.844519:0:14694:0:(nrs.c:1006:nrs_svcpt_setup_locked()) Process entered 00000100:00000001:0.0:1773941498.844521:0:14694:0:(nrs.c:924:nrs_register_policies_locked()) Process entered 00000100:00000001:0.0:1773941498.844524:0:14694:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941498.844528:0:14694:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d04fa29c0. 00000100:00000001:0.0:1773941498.844531:0:14694:0:(nrs.c:211:nrs_policy_start_locked()) Process entered 00000100:00000010:0.0:1773941498.844535:0:14694:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at ffff988d10872f00. 00000100:00000001:0.0:1773941498.844538:0:14694:0:(nrs.c:336:nrs_policy_start_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.844540:0:14694:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.844542:0:14694:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941498.844544:0:14694:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d04fa2480. 00000100:00000001:0.0:1773941498.844547:0:14694:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.844550:0:14694:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941498.844552:0:14694:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d04fa2180. 00000100:00000001:0.0:1773941498.844554:0:14694:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.844556:0:14694:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941498.844558:0:14694:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d04fa2240. 00000100:00000001:0.0:1773941498.844560:0:14694:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.844562:0:14694:0:(nrs.c:944:nrs_register_policies_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.844564:0:14694:0:(nrs.c:985:nrs_svcpt_setup_locked0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.844567:0:14694:0:(nrs.c:1022:nrs_svcpt_setup_locked()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941498.844569:0:14694:0:(nrs.c:1034:nrs_svcpt_setup_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.844571:0:14694:0:(nrs.c:1006:nrs_svcpt_setup_locked()) Process entered 00000100:00000001:0.0:1773941498.844577:0:14694:0:(nrs.c:924:nrs_register_policies_locked()) Process entered 00000100:00000001:0.0:1773941498.844579:0:14694:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941498.844582:0:14694:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d04fac180. 00000100:00000001:0.0:1773941498.844584:0:14694:0:(nrs.c:211:nrs_policy_start_locked()) Process entered 00000100:00000010:0.0:1773941498.844586:0:14694:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at ffff988d10872a40. 00000100:00000001:0.0:1773941498.844588:0:14694:0:(nrs.c:336:nrs_policy_start_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.844590:0:14694:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.844592:0:14694:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941498.844595:0:14694:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d04fac780. 00000100:00000001:0.0:1773941498.844597:0:14694:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.844599:0:14694:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941498.844601:0:14694:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d04facb40. 00000100:00000001:0.0:1773941498.844603:0:14694:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.844605:0:14694:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941498.844607:0:14694:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d04fac6c0. 00000100:00000001:0.0:1773941498.844609:0:14694:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.844611:0:14694:0:(nrs.c:944:nrs_register_policies_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.844613:0:14694:0:(nrs.c:985:nrs_svcpt_setup_locked0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.844615:0:14694:0:(nrs.c:1022:nrs_svcpt_setup_locked()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941498.844616:0:14694:0:(nrs.c:1034:nrs_svcpt_setup_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.844656:0:14694:0:(nrs.c:1379:ptlrpc_service_nrs_setup()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1773941498.844659:0:14694:0:(service.c:848:ptlrpc_register_service()) ldlm_cbd: Started, listening on portal 15 00000100:00000001:0.0:1773941498.844662:0:14694:0:(service.c:3400:ptlrpc_start_threads()) Process entered 00000100:00000001:0.0:1773941498.844664:0:14694:0:(service.c:3433:ptlrpc_start_thread()) Process entered 00000100:00100000:0.0:1773941498.844666:0:14694:0:(service.c:3441:ptlrpc_start_thread()) ldlm_cbd[0] started 0 min 2 max 34 00000100:00000010:0.0:1773941498.844671:0:14694:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff988d11267600. 00000100:00100000:0.0:1773941498.844674:0:14694:0:(service.c:3500:ptlrpc_start_thread()) starting thread 'ldlm_cb00_000' 00000100:00000001:0.0:1773941498.845130:0:14695:0:(service.c:2998:ptlrpc_main()) Process entered 00000100:00000010:0.0:1773941498.845151:0:14695:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff988d10872080. 00000020:00000010:0.0:1773941498.845163:0:14695:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff988d10872700. 00000020:00000010:0.0:1773941498.845170:0:14695:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d11267c00. 00000020:00000010:0.0:1773941498.845173:0:14695:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d11267a00. 00000040:00000010:0.0:1773941498.845177:0:14695:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d11266a00. 00000020:00000010:0.0:1773941498.845180:0:14695:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d111b2800. 00000020:00000010:0.0:1773941498.845185:0:14695:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff988d054eeeb0. 00000020:00000010:0.0:1773941498.845190:0:14695:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d0831ba00. 80000000:00000010:0.0:1773941498.845200:0:14695:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d06580060. 40000000:00000010:0.0:1773941498.845204:0:14695:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff988d10872140. 00100000:00000010:0.0:1773941498.845209:0:14695:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff988d111b5000. 00040000:00000010:0.0:1773941498.845229:0:14695:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d04c1a100. 00040000:00000010:0.0:1773941498.845235:0:14695:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff988d111b1000. 00040000:00000010:0.0:1773941498.845239:0:14695:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d02c84c00. 00000004:00000010:0.0:1773941498.845244:0:14695:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff988d111b6000. 00000004:00000010:0.0:1773941498.845250:0:14695:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff988d18a23000. 00000004:00000010:0.0:1773941498.845255:0:14695:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff988d111b3800. 00002000:00000010:0.0:1773941498.845260:0:14695:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff988d1081d400. 00000004:00000010:0.0:1773941498.845265:0:14695:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff988d1081e000. 00000100:00000200:0.0:1773941498.845269:0:14695:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_cbd: registering portal 15 00000400:00000010:0.0:1773941498.845275:0:14695:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037fbb0. 00000400:00000010:0.0:1773941498.845281:0:14695:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d04facc00. 00000100:00000010:0.0:1773941498.845289:0:14695:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff988d180dc800. 00000100:00000200:0.0:1773941498.845302:0:14695:0:(service.c:3086:ptlrpc_main()) service thread 0 (#1) started 00000100:00000001:3.0:1773941498.845394:0:14694:0:(service.c:3528:ptlrpc_start_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941498.845399:0:14694:0:(service.c:3433:ptlrpc_start_thread()) Process entered 00000100:00100000:3.0:1773941498.845401:0:14694:0:(service.c:3441:ptlrpc_start_thread()) ldlm_cbd[0] started 1 min 2 max 34 00000100:00000010:3.0:1773941498.845407:0:14694:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff988d1817c200. 00000100:00100000:3.0:1773941498.845411:0:14694:0:(service.c:3500:ptlrpc_start_thread()) starting thread 'ldlm_cb00_001' 00000100:00000001:0.0:1773941498.845591:0:14696:0:(service.c:2998:ptlrpc_main()) Process entered 00000100:00000010:0.0:1773941498.845608:0:14696:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff988d108722c0. 00000020:00000010:0.0:1773941498.845611:0:14696:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff988d10872300. 00000020:00000010:0.0:1773941498.845615:0:14696:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d02c84600. 00000020:00000010:0.0:1773941498.845618:0:14696:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d0a06d200. 00000040:00000010:0.0:1773941498.845620:0:14696:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d0a06d600. 00000020:00000010:0.0:1773941498.845623:0:14696:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d08af4800. 00000020:00000010:0.0:1773941498.845626:0:14696:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff988d054edf98. 00000020:00000010:0.0:1773941498.845628:0:14696:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d0a06d800. 80000000:00000010:0.0:1773941498.845630:0:14696:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d06580660. 40000000:00000010:0.0:1773941498.845632:0:14696:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff988d108721c0. 00100000:00000010:0.0:1773941498.845634:0:14696:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff988d08af3000. 00040000:00000010:0.0:1773941498.845637:0:14696:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d04c1a600. 00040000:00000010:0.0:1773941498.845639:0:14696:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff988d08af6000. 00040000:00000010:0.0:1773941498.845642:0:14696:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d063a9400. 00000004:00000010:0.0:1773941498.845644:0:14696:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff988d08af4000. 00000004:00000010:0.0:1773941498.845647:0:14696:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff988d034b4000. 00000004:00000010:0.0:1773941498.845649:0:14696:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff988d08af7000. 00002000:00000010:0.0:1773941498.845656:0:14696:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff988d1081dc00. 00000004:00000010:0.0:1773941498.845658:0:14696:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff988d1081e400. 00000100:00000010:0.0:1773941498.845661:0:14696:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff988d08af3800. 00000100:00000200:0.0:1773941498.845672:0:14696:0:(service.c:3086:ptlrpc_main()) service thread 1 (#2) started 00000100:00000001:3.0:1773941498.845685:0:14694:0:(service.c:3528:ptlrpc_start_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941498.845688:0:14694:0:(service.c:3433:ptlrpc_start_thread()) Process entered 00000100:00100000:3.0:1773941498.845689:0:14694:0:(service.c:3441:ptlrpc_start_thread()) ldlm_cbd[1] started 0 min 2 max 34 00000100:00000010:3.0:1773941498.845693:0:14694:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff988d1817d800. 00000100:00100000:3.0:1773941498.845695:0:14694:0:(service.c:3500:ptlrpc_start_thread()) starting thread 'ldlm_cb01_000' 00000800:00000010:1.0:1773941498.845935:0:2389:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4712 at ffff988d18bfc000. 00000800:00000010:1.0:1773941498.845942:0:2389:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at ffff988d08aaf6c0. 00000800:00000010:1.0:1773941498.845947:0:2389:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at ffff988d04d67300. 00000800:00000010:1.0:1773941498.845956:0:2389:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at ffff988d04d67300. 00000100:00000001:3.0:1773941498.846011:0:14697:0:(service.c:2998:ptlrpc_main()) Process entered 00000100:00000010:3.0:1773941498.846024:0:14697:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff988d03ec7a80. 00000020:00000010:3.0:1773941498.846027:0:14697:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff988d03ec7840. 00000020:00000010:3.0:1773941498.846030:0:14697:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d1817c400. 00000020:00000010:3.0:1773941498.846036:0:14697:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d1817c600. 00000040:00000010:3.0:1773941498.846038:0:14697:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d1817c000. 00000020:00000010:3.0:1773941498.846041:0:14697:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d07233000. 00000020:00000010:3.0:1773941498.846044:0:14697:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff988d03341e30. 00000020:00000010:3.0:1773941498.846046:0:14697:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d1817cc00. 80000000:00000010:3.0:1773941498.846048:0:14697:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d111f7ae0. 40000000:00000010:3.0:1773941498.846050:0:14697:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff988d03ec7c00. 00100000:00000010:3.0:1773941498.846052:0:14697:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff988d07230800. 00040000:00000010:3.0:1773941498.846055:0:14697:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d013d4000. 00040000:00000010:3.0:1773941498.846057:0:14697:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff988d07234800. 00040000:00000010:3.0:1773941498.846059:0:14697:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d1817ce00. 00000004:00000010:3.0:1773941498.846061:0:14697:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff988d07236800. 00000004:00000010:3.0:1773941498.846064:0:14697:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff988d02d6f000. 00000004:00000010:3.0:1773941498.846066:0:14697:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff988d07237000. 00002000:00000010:3.0:1773941498.846068:0:14697:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff988d03522000. 00000004:00000010:3.0:1773941498.846070:0:14697:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff988d03521400. 00000100:00000200:3.0:1773941498.846072:0:14697:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_cbd: registering portal 15 00000400:00000010:3.0:1773941498.846075:0:14697:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccddc0. 00000400:00000010:3.0:1773941498.846078:0:14697:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1837de40. 00000100:00000010:3.0:1773941498.846082:0:14697:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff988d0835a800. 00000100:00000200:3.0:1773941498.846094:0:14697:0:(service.c:3086:ptlrpc_main()) service thread 0 (#1) started 00000100:00000001:0.0:1773941498.846120:0:14694:0:(service.c:3528:ptlrpc_start_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.846127:0:14694:0:(service.c:3433:ptlrpc_start_thread()) Process entered 00000100:00100000:0.0:1773941498.846129:0:14694:0:(service.c:3441:ptlrpc_start_thread()) ldlm_cbd[1] started 1 min 2 max 34 00000100:00000010:0.0:1773941498.846135:0:14694:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff988d063a9000. 00000100:00100000:0.0:1773941498.846138:0:14694:0:(service.c:3500:ptlrpc_start_thread()) starting thread 'ldlm_cb01_001' 00000100:00000001:3.0:1773941498.846374:0:14698:0:(service.c:2998:ptlrpc_main()) Process entered 00000100:00000010:3.0:1773941498.846385:0:14698:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff988d03ec7900. 00000020:00000010:3.0:1773941498.846387:0:14698:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff988d03ec7180. 00000020:00000010:3.0:1773941498.846390:0:14698:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d1817dc00. 00000020:00000010:3.0:1773941498.846392:0:14698:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d1817c800. 00000040:00000010:3.0:1773941498.846394:0:14698:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d1817d200. 00000020:00000010:3.0:1773941498.846397:0:14698:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d0835d800. 00000020:00000010:3.0:1773941498.846399:0:14698:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff988d033469a8. 00000020:00000010:3.0:1773941498.846401:0:14698:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d1817ca00. 80000000:00000010:3.0:1773941498.846403:0:14698:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d111f7300. 40000000:00000010:3.0:1773941498.846405:0:14698:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff988d03ec7cc0. 00100000:00000010:3.0:1773941498.846407:0:14698:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff988d0835e800. 00040000:00000010:3.0:1773941498.846410:0:14698:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d21731a00. 00040000:00000010:3.0:1773941498.846416:0:14698:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff988d08359800. 00040000:00000010:3.0:1773941498.846418:0:14698:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d1817d000. 00000004:00000010:3.0:1773941498.846420:0:14698:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff988d08358800. 00000004:00000010:3.0:1773941498.846423:0:14698:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff988d02d6c000. 00000004:00000010:3.0:1773941498.846425:0:14698:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff988d08358000. 00002000:00000010:3.0:1773941498.846427:0:14698:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff988d03520800. 00000004:00000010:3.0:1773941498.846429:0:14698:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff988d03523c00. 00000100:00000010:3.0:1773941498.846432:0:14698:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff988d0835d000. 00000100:00000200:3.0:1773941498.846441:0:14698:0:(service.c:3086:ptlrpc_main()) service thread 1 (#2) started 00000100:00000001:0.0:1773941498.846453:0:14694:0:(service.c:3528:ptlrpc_start_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.846458:0:14694:0:(service.c:3418:ptlrpc_start_threads()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.846460:0:14694:0:(service.c:857:ptlrpc_register_service()) Process leaving (rc=18446630330378383360 : -113743331168256 : ffff988d11267000) 00000100:00000001:0.0:1773941498.846466:0:14694:0:(service.c:701:ptlrpc_register_service()) Process entered 00000100:00000010:0.0:1773941498.846470:0:14694:0:(service.c:745:ptlrpc_register_service()) kmalloced '(service)': 360 at ffff988d063a9c00. 00000100:00000010:0.0:1773941498.846474:0:14694:0:(service.c:803:ptlrpc_register_service()) kmalloced '(svcpt)': 784 at ffff988d1081c800. 00000100:00000010:0.0:1773941498.846490:0:14694:0:(service.c:627:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_array)': 12080 at ffff988d2036c000. 00000100:00000010:0.0:1773941498.846499:0:14694:0:(service.c:635:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_count)': 3020 at ffff988d07202000. 00000100:00000010:0.0:1773941498.846504:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1081e800. 00000100:00000010:0.0:1773941498.846509:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d10800000. 00000100:00000010:0.0:1773941498.846512:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1081cc00. 00000100:00000010:0.0:1773941498.846517:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d10804000. 00000100:00000010:0.0:1773941498.846519:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1081c400. 00000100:00000010:0.0:1773941498.846523:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08380000. 00000100:00000010:0.0:1773941498.846524:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1081c000. 00000100:00000010:0.0:1773941498.846528:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08384000. 00000100:00000010:0.0:1773941498.846529:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1081ec00. 00000100:00000010:0.0:1773941498.846533:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08386000. 00000100:00000010:0.0:1773941498.846534:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1081f400. 00000100:00000010:0.0:1773941498.846538:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d065e8000. 00000100:00000010:0.0:1773941498.846540:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1081f000. 00000100:00000010:0.0:1773941498.846543:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d065ea000. 00000100:00000010:0.0:1773941498.846545:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1081d000. 00000100:00000010:0.0:1773941498.846552:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d065ee000. 00000100:00000010:0.0:1773941498.846555:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1081f800. 00000100:00000010:0.0:1773941498.846558:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d065ec000. 00000100:00000010:0.0:1773941498.846560:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d113e1800. 00000100:00000010:0.0:1773941498.846564:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d107fc000. 00000100:00000010:0.0:1773941498.846566:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d113e2c00. 00000100:00000010:0.0:1773941498.846569:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d107f8000. 00000100:00000010:0.0:1773941498.846571:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d113e3c00. 00000100:00000010:0.0:1773941498.846574:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d107fa000. 00000100:00000010:0.0:1773941498.846576:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d113e1000. 00000100:00000010:0.0:1773941498.846579:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d107fe000. 00000100:00000010:0.0:1773941498.846581:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d113e2000. 00000100:00000010:0.0:1773941498.846585:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d0337a000. 00000100:00000010:0.0:1773941498.846588:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d113e0800. 00000100:00000010:0.0:1773941498.846591:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d0337e000. 00000100:00000010:0.0:1773941498.846593:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d113e1400. 00000100:00000010:0.0:1773941498.846596:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d0337c000. 00000100:00000010:0.0:1773941498.846599:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d113e3000. 00000100:00000010:0.0:1773941498.846602:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d03378000. 00000100:00000010:0.0:1773941498.846604:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d113e1c00. 00000100:00000010:0.0:1773941498.846608:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d07fd2000. 00000100:00000010:0.0:1773941498.846610:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d113e0c00. 00000100:00000010:0.0:1773941498.846613:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d07fd6000. 00000100:00000010:0.0:1773941498.846615:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d113e3800. 00000100:00000010:0.0:1773941498.846618:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d07fd4000. 00000100:00000010:0.0:1773941498.846620:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d113e2800. 00000100:00000010:0.0:1773941498.846623:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d07fd0000. 00000100:00000010:0.0:1773941498.846625:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d01683800. 00000100:00000010:0.0:1773941498.846628:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d20304000. 00000100:00000010:0.0:1773941498.846630:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d01681800. 00000100:00000010:0.0:1773941498.846633:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d20300000. 00000100:00000010:0.0:1773941498.846638:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d01682000. 00000100:00000010:0.0:1773941498.846641:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d20306000. 00000100:00000010:0.0:1773941498.846643:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d01681400. 00000100:00000010:0.0:1773941498.846646:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d20302000. 00000100:00000010:0.0:1773941498.846648:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d055a4400. 00000100:00000010:0.0:1773941498.846652:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d06222000. 00000100:00000010:0.0:1773941498.846653:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d055a4000. 00000100:00000010:0.0:1773941498.846656:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d06226000. 00000100:00000010:0.0:1773941498.846658:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d055a5400. 00000100:00000010:0.0:1773941498.846661:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d06224000. 00000100:00000010:0.0:1773941498.846663:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1078e400. 00000100:00000010:0.0:1773941498.846671:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d20324000. 00000100:00000010:0.0:1773941498.846673:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1078f400. 00000100:00000010:0.0:1773941498.846676:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d20320000. 00000100:00000010:0.0:1773941498.846678:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1078c000. 00000100:00000010:0.0:1773941498.846681:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d20322000. 00000100:00000010:0.0:1773941498.846683:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1078e800. 00000100:00000010:0.0:1773941498.846686:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d20326000. 00000100:00000010:0.0:1773941498.846687:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1078cc00. 00000100:00000010:0.0:1773941498.846694:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d065ba000. 00000100:00000010:0.0:1773941498.846696:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1078f800. 00000100:00000010:0.0:1773941498.846699:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d065be000. 00000100:00000010:0.0:1773941498.846701:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1078f000. 00000100:00000010:0.0:1773941498.846704:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d065bc000. 00000100:00000010:0.0:1773941498.846707:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1078dc00. 00000100:00000010:0.0:1773941498.846712:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d065b8000. 00000100:00000010:0.0:1773941498.846715:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1078d400. 00000100:00000010:0.0:1773941498.846725:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d05422000. 00000100:00000010:0.0:1773941498.846729:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1078e000. 00000100:00000010:0.0:1773941498.846733:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d05426000. 00000100:00000010:0.0:1773941498.846735:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1078c800. 00000100:00000010:0.0:1773941498.846744:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d05424000. 00000100:00000010:0.0:1773941498.846748:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1078fc00. 00000100:00000010:0.0:1773941498.846752:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d05420000. 00000100:00000010:0.0:1773941498.846756:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1078d000. 00000100:00000010:0.0:1773941498.846767:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08aea000. 00000100:00000010:0.0:1773941498.846772:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1078ec00. 00000100:00000010:0.0:1773941498.846777:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08aee000. 00000100:00000010:0.0:1773941498.846780:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1078c400. 00000100:00000010:0.0:1773941498.846784:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08aec000. 00000100:00000010:0.0:1773941498.846787:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1078d800. 00000100:00000010:0.0:1773941498.846792:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08ae8000. 00000100:00000010:0.0:1773941498.846796:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d19727800. 00000100:00000010:0.0:1773941498.846804:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d18a0e000. 00000100:00000010:0.0:1773941498.846806:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d19724400. 00000100:00000010:0.0:1773941498.846810:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d18a0c000. 00000100:00000010:0.0:1773941498.846812:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d107d3400. 00000100:00000010:0.0:1773941498.846816:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d18a08000. 00000100:00000010:0.0:1773941498.846818:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d107d2400. 00000100:00000010:0.0:1773941498.846822:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d18a0a000. 00000100:00000010:0.0:1773941498.846824:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d107d0400. 00000100:00000010:0.0:1773941498.846832:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08a2e000. 00000100:00000010:0.0:1773941498.846835:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d107d2c00. 00000100:00000010:0.0:1773941498.846839:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08a2c000. 00000400:00000200:1.0:1773941498.846844:0:2389:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000100:00000010:0.0:1773941498.846844:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d107d1800. 00000100:00000010:0.0:1773941498.846850:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08a28000. 00000800:00000200:1.1:1773941498.846853:0:2389:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 1, ksnr_connected 2 ksnr_max_conns 1 00000100:00000010:0.0:1773941498.846853:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d107d1000. 00000100:00000010:0.0:1773941498.846859:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08a2a000. 00000100:00000010:0.0:1773941498.846862:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d107d3c00. 00000800:00000200:1.1:1773941498.846867:0:2389:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000100:00000010:0.0:1773941498.846871:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d0713e000. 00000800:00000200:1.1:1773941498.846873:0:2389:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d1a2ec800 type 2, nob 96 niov 1 nkiov 0 00000100:00000010:0.0:1773941498.846875:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d107d0800. 00000100:00000010:0.0:1773941498.846883:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d0713c000. 00000800:00000200:1.0:1773941498.846886:0:2389:0:(socklnd.c:1183:ksocknal_create_conn()) New conn 12345-192.168.204.124@tcp p 3.x 192.168.204.14 -> 192.168.204.124:988 incarnation:1773941238881364903 sched[0] 00000100:00000010:0.0:1773941498.846889:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d107d2000. 00000800:00000010:1.0:1773941498.846895:0:2389:0:(socklnd.c:1191:ksocknal_create_conn()) kfreed 'hello': 144 at ffff988d08aaf6c0. 00000100:00000010:0.0:1773941498.846896:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d07138000. 00000100:00000010:0.0:1773941498.846901:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d107d1400. 00000100:00000010:0.0:1773941498.846905:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d0713a000. 00000100:00000010:0.0:1773941498.846908:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d107d1c00. 00000100:00000010:0.0:1773941498.846917:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d0357c000. 00000100:00000010:0.0:1773941498.846919:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d107d3000. 00000100:00000010:0.0:1773941498.846922:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d03578000. 00000100:00000010:0.0:1773941498.846924:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d107d3800. 00000100:00000010:0.0:1773941498.846928:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d0357a000. 00000100:00000010:0.0:1773941498.846930:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d107d0c00. 00000100:00000010:0.0:1773941498.846933:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d0357e000. 00000100:00000010:0.0:1773941498.846935:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d107d2800. 00000100:00000010:0.0:1773941498.846943:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d03502000. 00000100:00000010:0.0:1773941498.846946:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d107d0000. 00000100:00000010:0.0:1773941498.846950:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d03506000. 00000100:00000010:0.0:1773941498.846953:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d203fd800. 00000100:00000010:0.0:1773941498.846956:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d03504000. 00000100:00000010:0.0:1773941498.846984:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d203fd000. 00000100:00000010:0.0:1773941498.846991:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d03500000. 00000100:00100000:0.0:1773941498.846995:0:14694:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_canceld: allocate 64 new 8192-byte reqbufs (0/64 left), rc = 0 00000400:00000200:1.0:1773941498.846996:0:2389:0:(lib-socket.c:453:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.204.124:988 00000100:00000010:0.0:1773941498.847004:0:14694:0:(service.c:803:ptlrpc_register_service()) kmalloced '(svcpt)': 784 at ffff988d203ffc00. 00000100:00000010:0.0:1773941498.847017:0:14694:0:(service.c:627:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_array)': 12080 at ffff988d203f8000. 00000100:00000010:0.0:1773941498.847023:0:14694:0:(service.c:635:ptlrpc_service_part_init()) kmalloced '(array->paa_reqs_count)': 3020 at ffff988d07200000. 00000100:00000010:0.0:1773941498.847029:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d203fc400. 00000100:00000010:0.0:1773941498.847037:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d10816000. 00000100:00000010:0.0:1773941498.847042:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d203fec00. 00000100:00000010:0.0:1773941498.847046:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d10814000. 00000100:00000010:0.0:1773941498.847049:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d203ff400. 00000100:00000010:0.0:1773941498.847053:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d10810000. 00000100:00000010:0.0:1773941498.847056:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d203fe400. 00000100:00000010:0.0:1773941498.847064:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d10812000. 00000100:00000010:0.0:1773941498.847067:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d203fe800. 00000100:00000010:0.0:1773941498.847075:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d18a30000. 00000100:00000010:0.0:1773941498.847078:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d203fc000. 00000100:00000010:0.0:1773941498.847081:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d18a32000. 00000100:00000010:0.0:1773941498.847084:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d203fcc00. 00000100:00000010:0.0:1773941498.847087:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d18a36000. 00000100:00000010:0.0:1773941498.847090:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d203fe000. 00000100:00000010:0.0:1773941498.847093:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d18a34000. 00000100:00000010:0.0:1773941498.847095:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d203fd400. 00000800:00000001:1.0:1773941498.847098:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000100:00000010:0.0:1773941498.847103:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d089fa000. 00000800:00000001:1.0:1773941498.847106:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:0.0:1773941498.847107:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d203fdc00. 00000100:00000010:0.0:1773941498.847112:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d089fe000. 00000100:00000010:0.0:1773941498.847115:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d203fc800. 00000100:00000010:0.0:1773941498.847120:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d089fc000. 00000100:00000010:0.0:1773941498.847123:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d203ff800. 00000100:00000010:0.0:1773941498.847126:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d089f8000. 00000100:00000010:0.0:1773941498.847128:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d203ff000. 00000100:00000010:0.0:1773941498.847137:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08350000. 00000100:00000010:0.0:1773941498.847141:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1127f800. 00000100:00000010:0.0:1773941498.847145:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08352000. 00000100:00000010:0.0:1773941498.847148:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d065b1400. 00000100:00000010:0.0:1773941498.847151:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08356000. 00000100:00000010:0.0:1773941498.847154:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d065b0800. 00000800:00000200:1.0:1773941498.847155:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000010:0.0:1773941498.847157:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08354000. 00000800:00000010:1.0:1773941498.847161:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 232 at ffff988d1a2ec800. 00000100:00000010:0.0:1773941498.847162:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d065b2000. 00000400:00000200:1.0:1773941498.847169:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000010:0.0:1773941498.847173:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d089e0000. 00000100:00000010:0.0:1773941498.847176:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d065b1000. 00000400:00000200:1.0:1773941498.847180:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: GET: OK 00000100:00000010:0.0:1773941498.847182:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d089e2000. 00000100:00000010:0.0:1773941498.847185:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d065b3c00. 00000400:00000200:1.0:1773941498.847187:0:2393:0:(peer.c:3081:lnet_discovery_event_handler()) Received event: 5 00000100:00000010:0.0:1773941498.847189:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d089e6000. 00000400:00000200:1.0:1773941498.847193:0:2393:0:(peer.c:3033:lnet_discovery_event_send()) Ping Send to 192.168.204.124@tcp: 0 00000100:00000010:0.0:1773941498.847195:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d065b1800. 00000100:00000010:0.0:1773941498.847200:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d089e4000. 00000100:00000010:0.0:1773941498.847202:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d065b0400. 00000100:00000010:0.0:1773941498.847210:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d03322000. 00000100:00000010:0.0:1773941498.847226:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d065b2c00. 00000100:00000010:0.0:1773941498.847231:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d03326000. 00000100:00000010:0.0:1773941498.847233:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d065b3400. 00000100:00000010:0.0:1773941498.847236:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d03324000. 00000100:00000010:0.0:1773941498.847238:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d065b2400. 00000100:00000010:0.0:1773941498.847241:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d03320000. 00000100:00000010:0.0:1773941498.847243:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d065b2800. 00000100:00000010:0.0:1773941498.847250:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d111fe000. 00000100:00000010:0.0:1773941498.847252:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d065b0000. 00000100:00000010:0.0:1773941498.847256:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d111fc000. 00000100:00000010:0.0:1773941498.847258:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d065b0c00. 00000100:00000010:0.0:1773941498.847261:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d111f8000. 00000100:00000010:0.0:1773941498.847263:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d065b3000. 00000100:00000010:0.0:1773941498.847267:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d111fa000. 00000100:00000010:0.0:1773941498.847269:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d065b3800. 00000100:00000010:0.0:1773941498.847276:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d181ec000. 00000100:00000010:0.0:1773941498.847278:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d065b1c00. 00000100:00000010:0.0:1773941498.847282:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d181e8000. 00000100:00000010:0.0:1773941498.847284:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d1a052000. 00000100:00000010:0.0:1773941498.847287:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d181ea000. 00000100:00000010:0.0:1773941498.847289:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d181d0c00. 00000100:00000010:0.0:1773941498.847292:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d181ee000. 00000100:00000010:0.0:1773941498.847294:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d181d3000. 00000100:00000010:0.0:1773941498.847300:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d05458000. 00000100:00000010:0.0:1773941498.847302:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d181d3800. 00000100:00000010:0.0:1773941498.847305:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d0545a000. 00000100:00000010:0.0:1773941498.847307:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d181d0800. 00000100:00000010:0.0:1773941498.847313:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d0545e000. 00000100:00000010:0.0:1773941498.847315:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d181d2000. 00000100:00000010:0.0:1773941498.847318:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d0545c000. 00000100:00000010:0.0:1773941498.847320:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d181d1000. 00000100:00000010:0.0:1773941498.847327:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08a76000. 00000100:00000010:0.0:1773941498.847329:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d181d3c00. 00000100:00000010:0.0:1773941498.847332:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08a74000. 00000100:00000010:0.0:1773941498.847334:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d181d2800. 00000100:00000010:0.0:1773941498.847337:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08a70000. 00000100:00000010:0.0:1773941498.847339:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d181d2400. 00000100:00000010:0.0:1773941498.847342:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d08a72000. 00000100:00000010:0.0:1773941498.847343:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d181d3400. 00000100:00000010:0.0:1773941498.847350:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d111c6000. 00000100:00000010:0.0:1773941498.847352:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d181d2c00. 00000100:00000010:0.0:1773941498.847355:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d111c4000. 00000100:00000010:0.0:1773941498.847357:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d181d0000. 00000100:00000010:0.0:1773941498.847360:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d111c0000. 00000100:00000010:0.0:1773941498.847362:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d181d1400. 00000100:00000010:0.0:1773941498.847366:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d111c2000. 00000100:00000010:0.0:1773941498.847369:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d181d1800. 00000100:00000010:0.0:1773941498.847380:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d18b90000. 00000100:00000010:0.0:1773941498.847384:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d181d0400. 00000100:00000010:0.0:1773941498.847388:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d18b92000. 00000100:00000010:0.0:1773941498.847392:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d11273800. 00000100:00000010:0.0:1773941498.847397:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d18b96000. 00000100:00000010:0.0:1773941498.847402:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d0923dc00. 00000100:00000010:0.0:1773941498.847407:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d18b94000. 00000100:00000010:0.0:1773941498.847411:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d0923c000. 00000100:00000010:0.0:1773941498.847421:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d203a4000. 00000100:00000010:0.0:1773941498.847424:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d0923fc00. 00000100:00000010:0.0:1773941498.847428:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d203a0000. 00000100:00000010:0.0:1773941498.847434:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d0923c400. 00000100:00000010:0.0:1773941498.847437:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d203a2000. 00000100:00000010:0.0:1773941498.847440:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d0923d800. 00000100:00000010:0.0:1773941498.847445:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d203a6000. 00000100:00000010:0.0:1773941498.847448:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d0923c800. 00000100:00000010:0.0:1773941498.847456:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d18afa000. 00000100:00000010:0.0:1773941498.847460:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d0923f800. 00000100:00000010:0.0:1773941498.847465:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d18afe000. 00000100:00000010:0.0:1773941498.847467:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d0923cc00. 00000100:00000010:0.0:1773941498.847470:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d18afc000. 00000100:00000010:0.0:1773941498.847473:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d0923ec00. 00000100:00000010:0.0:1773941498.847476:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d18af8000. 00000100:00000010:0.0:1773941498.847479:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d0923e800. 00000100:00000010:0.0:1773941498.847487:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d065d2000. 00000800:00000010:1.0:1773941498.847490:0:2389:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4712 at ffff988d18bfa000. 00000100:00000010:0.0:1773941498.847491:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d0923e400. 00000100:00000010:0.0:1773941498.847495:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d065d6000. 00000100:00000010:0.0:1773941498.847497:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d0923d000. 00000100:00000010:0.0:1773941498.847500:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d065d4000. 00000100:00000010:0.0:1773941498.847502:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d0923d400. 00000800:00000010:1.0:1773941498.847508:0:2389:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at ffff988d08aaf6c0. 00000100:00000010:0.0:1773941498.847508:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d065d0000. 00000100:00000010:0.0:1773941498.847515:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d0923f400. 00000800:00000010:1.0:1773941498.847517:0:2389:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at ffff988d04d67000. 00000100:00000010:0.0:1773941498.847523:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d2031a000. 00000100:00000010:0.0:1773941498.847527:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d03074800. 00000800:00000010:1.0:1773941498.847529:0:2389:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at ffff988d04d67000. 00000100:00000010:0.0:1773941498.847533:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d2031e000. 00000100:00000010:0.0:1773941498.847538:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d02da7c00. 00000100:00000010:0.0:1773941498.847541:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d2031c000. 00000100:00000010:0.0:1773941498.847544:0:14694:0:(service.c:65:ptlrpc_alloc_rqbd()) kmalloced '(rqbd)': 968 at ffff988d11258c00. 00000100:00000010:0.0:1773941498.847547:0:14694:0:(service.c:75:ptlrpc_alloc_rqbd()) kmalloced '(rqbd->rqbd_buffer)': 8192 at ffff988d20318000. 00000100:00100000:0.0:1773941498.847548:0:14694:0:(service.c:153:ptlrpc_grow_req_bufs()) ldlm_canceld: allocate 64 new 8192-byte reqbufs (0/64 left), rc = 0 00000400:00000200:0.0:1773941498.847551:0:14694:0:(lib-ptl.c:921:LNetSetLazyPortal()) Setting portal 17 lazy 00000020:00000010:0.0:1773941498.847591:0:14694:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff988d11259400. 00000020:00000010:0.0:1773941498.847598:0:14694:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 3680 at ffff988d07207000. 00000100:00000001:0.0:1773941498.847632:0:14694:0:(nrs.c:1006:nrs_svcpt_setup_locked()) Process entered 00000100:00000001:0.0:1773941498.847634:0:14694:0:(nrs.c:924:nrs_register_policies_locked()) Process entered 00000100:00000001:0.0:1773941498.847635:0:14694:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941498.847638:0:14694:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d06510c00. 00000100:00000001:0.0:1773941498.847640:0:14694:0:(nrs.c:211:nrs_policy_start_locked()) Process entered 00000100:00000010:0.0:1773941498.847642:0:14694:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at ffff988d10872980. 00000100:00000001:0.0:1773941498.847644:0:14694:0:(nrs.c:336:nrs_policy_start_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.847646:0:14694:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.847647:0:14694:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941498.847649:0:14694:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d06510540. 00000100:00000001:0.0:1773941498.847650:0:14694:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.847652:0:14694:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941498.847654:0:14694:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d06510000. 00000100:00000001:0.0:1773941498.847655:0:14694:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.847656:0:14694:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941498.847657:0:14694:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d06510240. 00000100:00000001:0.0:1773941498.847658:0:14694:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.847660:0:14694:0:(nrs.c:944:nrs_register_policies_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.847661:0:14694:0:(nrs.c:985:nrs_svcpt_setup_locked0()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1773941498.847663:0:14694:0:(nrs.c:1026:nrs_svcpt_setup_locked()) kmalloced '(svcpt->scp_nrs_hp)': 112 at ffff988d181fef00. 00000100:00000001:0.0:1773941498.847664:0:14694:0:(nrs.c:924:nrs_register_policies_locked()) Process entered 00000100:00000001:0.0:1773941498.847665:0:14694:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941498.847666:0:14694:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d06510900. 00000100:00000001:0.0:1773941498.847667:0:14694:0:(nrs.c:211:nrs_policy_start_locked()) Process entered 00000100:00000010:0.0:1773941498.847668:0:14694:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at ffff988d10872e40. 00000100:00000001:0.0:1773941498.847669:0:14694:0:(nrs.c:336:nrs_policy_start_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.847670:0:14694:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.847671:0:14694:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941498.847672:0:14694:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d06510180. 00000100:00000001:0.0:1773941498.847673:0:14694:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.847674:0:14694:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941498.847675:0:14694:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d06510300. 00000100:00000001:0.0:1773941498.847676:0:14694:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.847677:0:14694:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941498.847678:0:14694:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d06510480. 00000100:00000001:0.0:1773941498.847679:0:14694:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.847683:0:14694:0:(nrs.c:944:nrs_register_policies_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.847684:0:14694:0:(nrs.c:985:nrs_svcpt_setup_locked0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.847686:0:14694:0:(nrs.c:1034:nrs_svcpt_setup_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.847687:0:14694:0:(nrs.c:1006:nrs_svcpt_setup_locked()) Process entered 00000100:00000001:0.0:1773941498.847688:0:14694:0:(nrs.c:924:nrs_register_policies_locked()) Process entered 00000100:00000001:0.0:1773941498.847689:0:14694:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941498.847690:0:14694:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d06510e40. 00000100:00000001:0.0:1773941498.847691:0:14694:0:(nrs.c:211:nrs_policy_start_locked()) Process entered 00000100:00000010:0.0:1773941498.847692:0:14694:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at ffff988d10872880. 00000100:00000001:0.0:1773941498.847693:0:14694:0:(nrs.c:336:nrs_policy_start_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.847694:0:14694:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.847695:0:14694:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941498.847696:0:14694:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d06510d80. 00000100:00000001:0.0:1773941498.847697:0:14694:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.847698:0:14694:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941498.847699:0:14694:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d06510cc0. 00000100:00000001:0.0:1773941498.847700:0:14694:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.847701:0:14694:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941498.847702:0:14694:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d065100c0. 00000100:00000001:0.0:1773941498.847703:0:14694:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.847704:0:14694:0:(nrs.c:944:nrs_register_policies_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.847705:0:14694:0:(nrs.c:985:nrs_svcpt_setup_locked0()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1773941498.847707:0:14694:0:(nrs.c:1026:nrs_svcpt_setup_locked()) kmalloced '(svcpt->scp_nrs_hp)': 112 at ffff988d181fe080. 00000100:00000001:0.0:1773941498.847709:0:14694:0:(nrs.c:924:nrs_register_policies_locked()) Process entered 00000100:00000001:0.0:1773941498.847710:0:14694:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941498.847711:0:14694:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d065106c0. 00000100:00000001:0.0:1773941498.847713:0:14694:0:(nrs.c:211:nrs_policy_start_locked()) Process entered 00000100:00000010:0.0:1773941498.847715:0:14694:0:(nrs_fifo.c:59:nrs_fifo_start()) kmalloced '(head)': 40 at ffff988d10872d00. 00000100:00000001:0.0:1773941498.847716:0:14694:0:(nrs.c:336:nrs_policy_start_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.847718:0:14694:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.847719:0:14694:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941498.847721:0:14694:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d06510f00. 00000100:00000001:0.0:1773941498.847723:0:14694:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.847724:0:14694:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941498.847726:0:14694:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d065103c0. 00000100:00000001:0.0:1773941498.847728:0:14694:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.847729:0:14694:0:(nrs.c:783:nrs_policy_register()) Process entered 00000100:00000010:0.0:1773941498.847736:0:14694:0:(nrs.c:794:nrs_policy_register()) kmalloced '(policy)': 152 at ffff988d06510a80. 00000100:00000001:0.0:1773941498.847738:0:14694:0:(nrs.c:842:nrs_policy_register()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.847740:0:14694:0:(nrs.c:944:nrs_register_policies_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.847742:0:14694:0:(nrs.c:985:nrs_svcpt_setup_locked0()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.847744:0:14694:0:(nrs.c:1034:nrs_svcpt_setup_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.847779:0:14694:0:(nrs.c:1379:ptlrpc_service_nrs_setup()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:0.0:1773941498.847782:0:14694:0:(service.c:848:ptlrpc_register_service()) ldlm_canceld: Started, listening on portal 17 00000100:00000001:0.0:1773941498.847785:0:14694:0:(service.c:3400:ptlrpc_start_threads()) Process entered 00000100:00000001:0.0:1773941498.847786:0:14694:0:(service.c:3433:ptlrpc_start_thread()) Process entered 00000100:00100000:0.0:1773941498.847787:0:14694:0:(service.c:3441:ptlrpc_start_thread()) ldlm_canceld[0] started 0 min 3 max 34 00000100:00000010:0.0:1773941498.847789:0:14694:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff988d063a8000. 00000100:00100000:0.0:1773941498.847791:0:14694:0:(service.c:3500:ptlrpc_start_thread()) starting thread 'ldlm_cn00_000' 00000100:00000001:2.0:1773941498.847951:0:14699:0:(service.c:2998:ptlrpc_main()) Process entered 00000800:00000001:1.0:1773941498.848031:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941498.848040:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941498.848043:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941498.848047:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941498.848053:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941498.848061:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : REPLY - for me 00000400:00000200:1.0:1773941498.848069:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941498.848074:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941498.848084:0:2393:0:(lib-move.c:4647:lnet_parse_reply()) 192.168.204.14@tcp: Reply from 12345-192.168.204.124@tcp of length 48/48 into md 0x60d 00000800:00000001:1.0:1773941498.848091:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941498.848109:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941498.848112:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000010:0.0:1773941498.848113:0:14699:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff988d10872d80. 00000400:00000200:1.0:1773941498.848119:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: REPLY: OK 00000020:00000010:0.0:1773941498.848120:0:14699:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff988d10872cc0. 00000400:00000200:1.0:1773941498.848129:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac990 00000020:00000010:0.0:1773941498.848129:0:14699:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d1961e200. 00000400:00000010:1.0:1773941498.848134:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac990. 00000020:00000010:0.0:1773941498.848136:0:14699:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d1961fa00. 00000400:00000200:1.0:1773941498.848141:0:2393:0:(peer.c:3081:lnet_discovery_event_handler()) Received event: 3 00000020:00000010:0.0:1773941498.848143:0:14699:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d06580600. 00000400:00000200:1.0:1773941498.848148:0:2393:0:(peer.c:2911:lnet_discovery_event_reply()) Peer 192.168.204.124@tcp has discovery enabled 00000040:00000010:0.0:1773941498.848150:0:14699:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d1961ea00. 00000400:00000200:1.0:1773941498.848155:0:2393:0:(peer.c:2923:lnet_discovery_event_reply()) peer 192.168.204.124@tcp(ffff988d03357600) is MR 00000020:00000010:0.0:1773941498.848157:0:14699:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d08af5800. 00000400:00000200:1.0:1773941498.848162:0:2393:0:(peer.c:2992:lnet_discovery_event_reply()) peer 192.168.204.124@tcp data present 363512030. state = 0x241 00000020:00000010:0.0:1773941498.848164:0:14699:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff988d054ead48. 00000400:00000200:1.0:1773941498.848169:0:2393:0:(peer.c:2405:lnet_peer_queue_for_discovery()) Queue peer 192.168.204.124@tcp: -114 00000020:00000010:0.0:1773941498.848173:0:14699:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d217b6600. 80000000:00000010:0.0:1773941498.848178:0:14699:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d065801e0. 40000000:00000010:0.0:1773941498.848182:0:14699:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff988d10872780. 00000800:00000001:1.0:1773941498.848188:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00008000:00000010:0.0:1773941498.848188:0:14699:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d03335c40. 00000400:00000200:3.0:1773941498.848193:0:2399:0:(peer.c:3988:lnet_peer_discovery_wait_for_work()) woken: 0 00000800:00000001:1.0:1773941498.848194:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00100000:00000010:0.0:1773941498.848195:0:14699:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff988d20330000. 00040000:00000010:0.0:1773941498.848201:0:14699:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d02d87000. 00000400:00000200:3.0:1773941498.848203:0:2399:0:(peer.c:4100:lnet_peer_discovery()) peer 192.168.204.124@tcp(ffff988d03357600) state 0xc1 00040000:00000010:0.0:1773941498.848205:0:14699:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff988d20331000. 00040000:00000010:0.0:1773941498.848210:0:14699:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d10401000. 00000004:00000010:0.0:1773941498.848226:0:14699:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff988d20332800. 00000400:00000200:3.0:1773941498.848231:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000004:00000010:0.0:1773941498.848232:0:14699:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff988d07205000. 00000004:00000010:0.0:1773941498.848237:0:14699:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff988d20335800. 00002000:00000010:0.0:1773941498.848239:0:14699:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff988d1125b800. 00000400:00000200:3.0:1773941498.848240:0:2399:0:(peer.c:432:lnet_peer_del_locked()) peer 192.168.204.114@tcp 00000004:00000010:0.0:1773941498.848242:0:14699:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff988d1125b000. 00000100:00000200:0.0:1773941498.848246:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000200:3.0:1773941498.848249:0:2399:0:(peer.c:354:lnet_peer_detach_peer_ni_locked()) peer 192.168.204.114@tcp NID 192.168.204.114@tcp 00000400:00000010:0.0:1773941498.848255:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f210. 00000400:00000010:0.0:1773941498.848259:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d06510b40. 00000100:00000200:0.0:1773941498.848262:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848263:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f528. 00000400:00000010:0.0:1773941498.848265:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d06510780. 00000100:00000200:0.0:1773941498.848266:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848267:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f268. 00000400:00000010:0.0:1773941498.848269:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d2081a480. 00000100:00000200:0.0:1773941498.848270:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848272:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f058. 00000400:00000010:0.0:1773941498.848273:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d2081a0c0. 00000100:00000200:0.0:1773941498.848274:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848275:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037fec8. 00000400:00000010:0.0:1773941498.848277:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d2081ad80. 00000100:00000200:0.0:1773941498.848278:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848280:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f2c0. 00000400:00000010:0.0:1773941498.848282:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d034cc240. 00000100:00000200:0.0:1773941498.848283:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848284:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f948. 00000400:00000010:0.0:1773941498.848287:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d04c81840. 00000100:00000200:0.0:1773941498.848288:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848290:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f9a0. 00000400:00000010:0.0:1773941498.848292:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d04c810c0. 00000100:00000200:0.0:1773941498.848293:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848295:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f4d0. 00000400:00000010:0.0:1773941498.848298:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d20863300. 00000100:00000200:0.0:1773941498.848299:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848302:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037fc08. 00000400:00000010:0.0:1773941498.848304:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d20863180. 00000100:00000200:0.0:1773941498.848305:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848306:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f630. 00000400:00000010:0.0:1773941498.848307:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d20863840. 00000100:00000200:0.0:1773941498.848309:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848311:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f9f8. 00000400:00000010:0.0:1773941498.848312:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d20863e40. 00000100:00000200:0.0:1773941498.848318:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848319:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f318. 00000400:00000010:0.0:1773941498.848322:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d20863a80. 00000100:00000200:0.0:1773941498.848324:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000200:3.0:1773941498.848327:0:2399:0:(peer.c:2187:lnet_destroy_peer_ni_locked()) ffff988d0822f000 nid 192.168.204.114@tcp 00000400:00000010:0.0:1773941498.848328:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037fe70. 00000400:00000010:0.0:1773941498.848331:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d20863540. 00000400:00000010:3.0:1773941498.848334:0:2399:0:(peer.c:2217:lnet_destroy_peer_ni_locked()) kfreed 'lpni': 384 at ffff988d0822f000. 00000100:00000200:0.0:1773941498.848335:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848340:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f898. 00000400:00000010:0.0:1773941498.848344:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d20863900. 00000400:00000200:3.0:1773941498.848345:0:2399:0:(peer.c:212:lnet_destroy_peer_net_locked()) ffff988d183028a0 net tcp 00000100:00000200:0.0:1773941498.848347:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941498.848350:0:2399:0:(peer.c:219:lnet_destroy_peer_net_locked()) kfreed 'lpn': 72 at ffff988d183028a0. 00000400:00000010:0.0:1773941498.848351:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037fb00. 00000400:00000010:0.0:1773941498.848358:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d20863480. 00000400:00000200:3.0:1773941498.848360:0:2399:0:(peer.c:270:lnet_destroy_peer_locked()) ffff988d1a204a00 nid 192.168.204.114@tcp 00000100:00000200:0.0:1773941498.848362:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941498.848366:0:2399:0:(peer.c:300:lnet_destroy_peer_locked()) kfreed 'lp': 376 at ffff988d1a204a00. 00000400:00000010:0.0:1773941498.848368:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037fe18. 00000400:00000200:1.0:1773941498.848371:0:2389:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000010:0.0:1773941498.848373:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d208639c0. 00000100:00000200:0.0:1773941498.848377:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000800:00000200:1.1:1773941498.848378:0:2389:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 2, ksnr_connected 6 ksnr_max_conns 1 00000400:00000010:0.0:1773941498.848380:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f1b8. 00000400:00000010:0.0:1773941498.848385:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d20863f00. 00000800:00000200:1.0:1773941498.848389:0:2389:0:(socklnd.c:1183:ksocknal_create_conn()) New conn 12345-192.168.204.124@tcp p 3.x 192.168.204.14 -> 192.168.204.124:988 incarnation:1773941238881364903 sched[0] 00000100:00000200:0.0:1773941498.848390:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848395:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037fc60. 00000800:00000010:1.0:1773941498.848398:0:2389:0:(socklnd.c:1191:ksocknal_create_conn()) kfreed 'hello': 144 at ffff988d08aaf6c0. 00000400:00000010:0.0:1773941498.848400:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d20863780. 00000100:00000200:0.0:1773941498.848406:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848409:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f000. 00000400:00000200:3.0:1773941498.848416:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941498.848416:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a03cf00. 00000100:00000200:0.0:1773941498.848421:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941498.848422:0:2399:0:(peer.c:137:lnet_peer_ni_alloc()) alloc '(lpni)': 384 at ffff988d1817da00. 00000400:00000010:0.0:1773941498.848427:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037fcb8. 00000400:00000200:3.0:1773941498.848428:0:2399:0:(peer.c:183:lnet_peer_ni_alloc()) ffff988d1817da00 nid 192.168.204.114@tcp 00000400:00000010:0.0:1773941498.848436:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a03c180. 00000400:00000200:3.0:1773941498.848439:0:2399:0:(peer.c:1639:lnet_peer_attach_peer_ni()) peer 192.168.204.114@tcp NID 192.168.204.114@tcp flags 0x11 00000100:00000200:0.0:1773941498.848442:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000200:1.0:1773941498.848445:0:2389:0:(lib-socket.c:453:lnet_sock_connect()) Error -99 connecting 1023 -> 192.168.204.124:988 00000400:00000200:3.0:1773941498.848447:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000010:0.0:1773941498.848448:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f6e0. 00000400:00000200:3.0:1773941498.848453:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000010:0.0:1773941498.848455:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a03ccc0. 00000100:00000200:0.0:1773941498.848460:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000200:3.0:1773941498.848461:0:2399:0:(peer.c:1899:lnet_peer_add_nid()) peer 192.168.204.114@tcp NID 192.168.204.124@tcp flags 0x11: 0 00000400:00000010:0.0:1773941498.848466:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037fb58. 00000400:00000010:3.0:1773941498.848470:0:2399:0:(peer.c:3265:lnet_peer_merge_data()) alloc '(curnis)': 40 at ffff988d03ec75c0. 00000400:00000010:0.0:1773941498.848471:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a03c600. 00000100:00000200:0.0:1773941498.848477:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:3.0:1773941498.848480:0:2399:0:(peer.c:3266:lnet_peer_merge_data()) alloc '(addnis)': 48 at ffff988d03ec7080. 00000400:00000010:0.0:1773941498.848480:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037ff20. 00000400:00000010:3.0:1773941498.848486:0:2399:0:(peer.c:3267:lnet_peer_merge_data()) alloc '(delnis)': 40 at ffff988d03ec7600. 00000400:00000010:0.0:1773941498.848490:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a03c240. 00000400:00000200:1.0:1773941498.848491:0:2389:0:(lib-socket.c:453:lnet_sock_connect()) Error -99 connecting 1022 -> 192.168.204.124:988 00000400:00000200:3.0:1773941498.848496:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000100:00000200:0.0:1773941498.848496:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848503:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037fd68. 00000400:00000010:0.0:1773941498.848506:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a03c840. 00000400:00000200:3.0:1773941498.848511:0:2399:0:(peer.c:566:lnet_peer_del_nid()) peer 192.168.204.114@tcp NID 192.168.204.114@tcp flags 0x11: -1 00000100:00000200:0.0:1773941498.848511:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848516:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f8f0. 00000400:00020000:3.0:1773941498.848522:0:2399:0:(peer.c:3365:lnet_peer_merge_data()) Error deleting NID 192.168.204.114@tcp from peer 192.168.204.114@tcp: -1 00000400:00000010:0.0:1773941498.848524:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a0333c0. 00000100:00000200:0.0:1773941498.848527:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848541:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f3c8. 00000400:00000010:0.0:1773941498.848545:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a033cc0. 00000100:00000200:0.0:1773941498.848546:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848548:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037fdc0. 00000400:00000010:0.0:1773941498.848550:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a29d480. 00000100:00000200:0.0:1773941498.848552:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848553:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f0b0. 00000400:00000010:0.0:1773941498.848558:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a29df00. 00000100:00000200:0.0:1773941498.848560:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848561:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f738. 00000400:00000010:0.0:1773941498.848564:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a29d0c0. 00000100:00000200:0.0:1773941498.848565:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848567:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f478. 00000400:00000010:0.0:1773941498.848569:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a29dcc0. 00000800:00000001:1.0:1773941498.848571:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000100:00000200:0.0:1773941498.848571:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848576:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f688. 00000400:00000010:0.0:1773941498.848579:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a29d300. 00000800:00000001:1.0:1773941498.848580:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:0.0:1773941498.848581:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848586:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f420. 00000400:00000010:0.0:1773941498.848588:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a29d900. 00000100:00000200:0.0:1773941498.848589:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848590:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f790. 00000400:00000010:0.0:1773941498.848592:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a29d9c0. 00000100:00000200:0.0:1773941498.848593:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848594:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037ff78. 00000400:00000010:0.0:1773941498.848596:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d063e4300. 00000100:00000200:0.0:1773941498.848597:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848602:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037fa50. 00000400:00000010:0.0:1773941498.848604:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d063e4840. 00000100:00000200:0.0:1773941498.848606:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848607:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f108. 00000400:00000010:0.0:1773941498.848608:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d063e4d80. 00000100:00000200:0.0:1773941498.848610:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848611:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037faa8. 00000400:00000010:0.0:1773941498.848612:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d063e46c0. 00000100:00000200:0.0:1773941498.848613:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848614:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f160. 00000400:00000010:0.0:1773941498.848616:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d063e4540. 00000100:00000200:0.0:1773941498.848617:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848618:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037fd10. 00000400:00000010:0.0:1773941498.848620:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d08e34600. 00000100:00000200:0.0:1773941498.848621:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848622:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f370. 00000400:00000010:0.0:1773941498.848624:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d08e34300. 00000100:00000200:0.0:1773941498.848625:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848626:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f7e8. 00000400:00000010:0.0:1773941498.848627:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d08e34e40. 00000100:00000200:0.0:1773941498.848628:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848629:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f840. 00000400:00000010:0.0:1773941498.848631:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d08e34900. 00000100:00000200:0.0:1773941498.848632:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848634:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f5d8. 00000400:00000010:0.0:1773941498.848635:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d08e34f00. 00000100:00000200:0.0:1773941498.848636:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848637:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d2037f580. 00000400:00000010:0.0:1773941498.848639:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d08e34cc0. 00000100:00000200:0.0:1773941498.848640:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848642:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d1121eaa8. 00000400:00000010:0.0:1773941498.848643:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d08e34840. 00000100:00000200:0.0:1773941498.848644:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848646:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d1121e160. 00000400:00000010:0.0:1773941498.848647:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d08e343c0. 00000100:00000200:0.0:1773941498.848652:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848654:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d1121ed10. 00000400:00000010:0.0:1773941498.848656:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d08fd3180. 00000100:00000200:0.0:1773941498.848658:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848659:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d1121e580. 00000400:00000010:0.0:1773941498.848661:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d08fd39c0. 00000100:00000200:0.0:1773941498.848662:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848663:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d1121e5d8. 00000400:00000010:0.0:1773941498.848664:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d08fd3f00. 00000100:00000200:0.0:1773941498.848666:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848667:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d1121e7e8. 00000400:00000010:0.0:1773941498.848669:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d08feb480. 00000100:00000200:0.0:1773941498.848670:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848671:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d1121e840. 00000400:00000010:0.0:1773941498.848673:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d08feb6c0. 00000100:00000200:0.0:1773941498.848674:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848675:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d1121e370. 00000400:00000010:0.0:1773941498.848676:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d08febe40. 00000100:00000200:0.0:1773941498.848677:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848679:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d1121e790. 00000400:00000010:0.0:1773941498.848680:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d08feba80. 00000100:00000200:0.0:1773941498.848681:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848682:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d1121e420. 00000400:00000010:0.0:1773941498.848684:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d08feb240. 00000100:00000200:0.0:1773941498.848685:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848686:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d1121ebb0. 00000400:00000010:0.0:1773941498.848687:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d08feb9c0. 00000100:00000200:0.0:1773941498.848688:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848689:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d1121e210. 00000400:00000010:0.0:1773941498.848690:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d08febf00. 00000100:00000200:0.0:1773941498.848692:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848693:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d1121e528. 00000400:00000010:0.0:1773941498.848694:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d08feb0c0. 00000100:00000200:0.0:1773941498.848695:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848696:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d1121e268. 00000400:00000010:0.0:1773941498.848700:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d08febc00. 00000100:00000200:0.0:1773941498.848702:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848704:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d1121e058. 00000400:00000010:0.0:1773941498.848705:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d08febb40. 00000100:00000200:0.0:1773941498.848707:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848708:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d1121eec8. 00000400:00000010:0.0:1773941498.848710:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d08febd80. 00000100:00000200:0.0:1773941498.848713:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848715:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d1121ef78. 00000400:00000010:0.0:1773941498.848718:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d2027e000. 00000100:00000200:0.0:1773941498.848720:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848722:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d1121ea50. 00000400:00000010:0.0:1773941498.848725:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d2027ecc0. 00000100:00000200:0.0:1773941498.848727:0:14699:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:0.0:1773941498.848729:0:14699:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0014b108. 00000400:00000010:0.0:1773941498.848732:0:14699:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d2027ea80. 00000100:00000010:0.0:1773941498.848736:0:14699:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff988d20334800. 00000100:00000200:0.0:1773941498.848750:0:14699:0:(service.c:3086:ptlrpc_main()) service thread 0 (#1) started 00000100:00000001:1.0:1773941498.848774:0:14694:0:(service.c:3528:ptlrpc_start_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.848778:0:14694:0:(service.c:3433:ptlrpc_start_thread()) Process entered 00000100:00100000:1.0:1773941498.848779:0:14694:0:(service.c:3441:ptlrpc_start_thread()) ldlm_canceld[0] started 1 min 3 max 34 00000100:00000010:1.0:1773941498.848785:0:14694:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff988d08a4fc00. 00000100:00100000:1.0:1773941498.848788:0:14694:0:(service.c:3500:ptlrpc_start_thread()) starting thread 'ldlm_cn00_001' 00000100:00000001:1.0:1773941498.848942:0:14700:0:(service.c:2998:ptlrpc_main()) Process entered 00000100:00000010:1.0:1773941498.848990:0:14700:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff988d04d67540. 00000020:00000010:1.0:1773941498.848996:0:14700:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff988d04d67b40. 00000020:00000010:1.0:1773941498.849001:0:14700:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d08a4f000. 00000020:00000010:1.0:1773941498.849005:0:14700:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d08a4f800. 00000020:00000010:1.0:1773941498.849009:0:14700:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d054f3c00. 00000040:00000010:1.0:1773941498.849012:0:14700:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d08a4e800. 00000020:00000010:1.0:1773941498.849016:0:14700:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d18b1e800. 00000020:00000010:1.0:1773941498.849020:0:14700:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff988d181f3250. 00000020:00000010:1.0:1773941498.849023:0:14700:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d08a4e400. 80000000:00000010:1.0:1773941498.849032:0:14700:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d054f3000. 40000000:00000010:1.0:1773941498.849035:0:14700:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff988d04d67480. 00008000:00000010:1.0:1773941498.849039:0:14700:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d0833b880. 00100000:00000010:1.0:1773941498.849042:0:14700:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff988d18b1d800. 00040000:00000010:1.0:1773941498.849045:0:14700:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d02cfd000. 00040000:00000010:1.0:1773941498.849048:0:14700:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff988d18b1c800. 00040000:00000010:1.0:1773941498.849051:0:14700:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d08a4e200. 00000004:00000010:1.0:1773941498.849055:0:14700:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff988d18b1b000. 00000004:00000010:1.0:1773941498.849059:0:14700:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff988d065f6000. 00000004:00000010:1.0:1773941498.849062:0:14700:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff988d18b1c000. 00002000:00000010:1.0:1773941498.849065:0:14700:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff988d11189800. 00000004:00000010:1.0:1773941498.849068:0:14700:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff988d1118ac00. 00000100:00000010:1.0:1773941498.849072:0:14700:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff988d18b1a000. 00000100:00000200:1.0:1773941498.849084:0:14700:0:(service.c:3086:ptlrpc_main()) service thread 1 (#2) started 00000100:00000001:0.0:1773941498.849090:0:14694:0:(service.c:3528:ptlrpc_start_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.849093:0:14694:0:(service.c:3433:ptlrpc_start_thread()) Process entered 00000100:00100000:0.0:1773941498.849094:0:14694:0:(service.c:3441:ptlrpc_start_thread()) ldlm_canceld[0] started 2 min 3 max 34 00000100:00000010:0.0:1773941498.849098:0:14694:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff988d10401c00. 00000100:00100000:0.0:1773941498.849101:0:14694:0:(service.c:3500:ptlrpc_start_thread()) starting thread 'ldlm_cn00_002' 00000100:00000001:0.0:1773941498.849277:0:14701:0:(service.c:2998:ptlrpc_main()) Process entered 00000100:00000010:0.0:1773941498.849288:0:14701:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff988d108724c0. 00000020:00000010:0.0:1773941498.849290:0:14701:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff988d10872280. 00000020:00000010:0.0:1773941498.849292:0:14701:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d10400c00. 00000020:00000010:0.0:1773941498.849294:0:14701:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d10401800. 00000020:00000010:0.0:1773941498.849296:0:14701:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d06580960. 00000040:00000010:0.0:1773941498.849298:0:14701:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d10400800. 00000020:00000010:0.0:1773941498.849300:0:14701:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d20330800. 00000020:00000010:0.0:1773941498.849302:0:14701:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff988d054ef3b8. 00000020:00000010:0.0:1773941498.849304:0:14701:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d10400400. 80000000:00000010:0.0:1773941498.849307:0:14701:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d06580f60. 40000000:00000010:0.0:1773941498.849308:0:14701:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff988d108725c0. 00008000:00000010:0.0:1773941498.849315:0:14701:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d03334710. 00100000:00000010:0.0:1773941498.849317:0:14701:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff988d20332000. 00040000:00000010:0.0:1773941498.849319:0:14701:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d02d87300. 00040000:00000010:0.0:1773941498.849321:0:14701:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff988d20336800. 00040000:00000010:0.0:1773941498.849323:0:14701:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d10401e00. 00000004:00000010:0.0:1773941498.849325:0:14701:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff988d20335000. 00000004:00000010:0.0:1773941498.849329:0:14701:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff988d02bc7000. 00000004:00000010:0.0:1773941498.849331:0:14701:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff988d20331800. 00002000:00000010:0.0:1773941498.849334:0:14701:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff988d11258000. 00000004:00000010:0.0:1773941498.849336:0:14701:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff988d1125a800. 00000100:00000010:0.0:1773941498.849338:0:14701:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff988d20333800. 00000100:00000200:0.0:1773941498.849348:0:14701:0:(service.c:3086:ptlrpc_main()) service thread 2 (#3) started 00000100:00000001:1.0:1773941498.849357:0:14694:0:(service.c:3528:ptlrpc_start_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.849362:0:14694:0:(service.c:3433:ptlrpc_start_thread()) Process entered 00000100:00100000:1.0:1773941498.849363:0:14694:0:(service.c:3441:ptlrpc_start_thread()) ldlm_canceld[1] started 0 min 3 max 34 00000100:00000010:1.0:1773941498.849368:0:14694:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff988d08a4ee00. 00000100:00100000:1.0:1773941498.849371:0:14694:0:(service.c:3500:ptlrpc_start_thread()) starting thread 'ldlm_cn01_000' 00000100:00000001:1.0:1773941498.849521:0:14702:0:(service.c:2998:ptlrpc_main()) Process entered 00000100:00000010:2.0:1773941498.849578:0:14702:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff988d05437340. 00000020:00000010:2.0:1773941498.849584:0:14702:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff988d05437040. 00000020:00000010:2.0:1773941498.849588:0:14702:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d0350e800. 00000020:00000010:2.0:1773941498.849591:0:14702:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d0350f400. 00000020:00000010:2.0:1773941498.849594:0:14702:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d08bbb960. 00000040:00000010:2.0:1773941498.849598:0:14702:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d0350e600. 00000020:00000010:2.0:1773941498.849607:0:14702:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d05449800. 00000020:00000010:2.0:1773941498.849611:0:14702:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff988d05400a10. 00000020:00000010:2.0:1773941498.849616:0:14702:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d0350fa00. 80000000:00000010:2.0:1773941498.849618:0:14702:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d08bbbd20. 40000000:00000010:2.0:1773941498.849620:0:14702:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff988d05437300. 00008000:00000010:2.0:1773941498.849622:0:14702:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d08324710. 00100000:00000010:2.0:1773941498.849625:0:14702:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff988d05449000. 00040000:00000010:2.0:1773941498.849627:0:14702:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d217a8d00. 00040000:00000010:2.0:1773941498.849629:0:14702:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff988d0544b800. 00040000:00000010:2.0:1773941498.849631:0:14702:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d0350e000. 00000004:00000010:2.0:1773941498.849633:0:14702:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff988d0544e000. 00000004:00000010:2.0:1773941498.849636:0:14702:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff988d08afb000. 00000004:00000010:2.0:1773941498.849638:0:14702:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff988d0544f800. 00002000:00000010:2.0:1773941498.849640:0:14702:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff988d035a6800. 00000004:00000010:2.0:1773941498.849642:0:14702:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff988d035a5800. 00000100:00000200:2.0:1773941498.849645:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849648:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e948. 00000400:00000010:2.0:1773941498.849651:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d02f7d3c0. 00000100:00000200:2.0:1773941498.849653:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849655:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e4d0. 00000400:00000010:2.0:1773941498.849657:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d02f7da80. 00000100:00000200:2.0:1773941498.849658:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849660:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380ec08. 00000400:00000010:2.0:1773941498.849661:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d02f7d300. 00000100:00000200:2.0:1773941498.849662:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849664:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e630. 00000400:00000010:2.0:1773941498.849670:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d02f7dd80. 00000100:00000200:2.0:1773941498.849672:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849674:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e9f8. 00000400:00000010:2.0:1773941498.849675:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d02f7d900. 00000100:00000200:2.0:1773941498.849677:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849678:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e318. 00000400:00000010:2.0:1773941498.849680:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d1a040840. 00000100:00000200:2.0:1773941498.849681:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849683:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380ee70. 00000400:00000010:2.0:1773941498.849684:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d0705b780. 00000100:00000200:2.0:1773941498.849685:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849687:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e898. 00000400:00000010:2.0:1773941498.849689:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d035dad80. 00000100:00000200:2.0:1773941498.849690:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849691:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380eb00. 00000400:00000010:2.0:1773941498.849692:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d035daa80. 00000100:00000200:2.0:1773941498.849694:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849695:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380ee18. 00000400:00000010:2.0:1773941498.849697:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d035dacc0. 00000100:00000200:2.0:1773941498.849698:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849699:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e1b8. 00000400:00000010:2.0:1773941498.849701:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d035da300. 00000100:00000200:2.0:1773941498.849702:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849703:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380ec60. 00000400:00000010:2.0:1773941498.849705:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d035da480. 00000100:00000200:2.0:1773941498.849707:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849709:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e000. 00000400:00000010:2.0:1773941498.849711:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d035da840. 00000100:00000200:2.0:1773941498.849713:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849716:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380ecb8. 00000400:00000010:2.0:1773941498.849718:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d035da0c0. 00000100:00000200:2.0:1773941498.849720:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849723:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e6e0. 00000400:00000010:2.0:1773941498.849725:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d035daf00. 00000100:00000200:2.0:1773941498.849727:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849734:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380eb58. 00000400:00000010:2.0:1773941498.849737:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d035dac00. 00000100:00000200:2.0:1773941498.849740:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849742:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380ef20. 00000400:00000010:2.0:1773941498.849745:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18807900. 00000100:00000200:2.0:1773941498.849748:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849750:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380ed68. 00000400:00000010:2.0:1773941498.849753:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18807d80. 00000100:00000200:2.0:1773941498.849755:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849757:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e8f0. 00000400:00000010:2.0:1773941498.849760:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18807c00. 00000100:00000200:2.0:1773941498.849762:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849765:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e3c8. 00000400:00000010:2.0:1773941498.849767:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18807a80. 00000100:00000200:2.0:1773941498.849769:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849772:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380edc0. 00000400:00000010:2.0:1773941498.849774:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d188073c0. 00000100:00000200:2.0:1773941498.849776:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849777:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e0b0. 00000400:00000010:2.0:1773941498.849779:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18807180. 00000100:00000200:2.0:1773941498.849780:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849782:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e738. 00000400:00000010:2.0:1773941498.849783:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d188079c0. 00000100:00000200:2.0:1773941498.849784:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849786:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e478. 00000400:00000010:2.0:1773941498.849787:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18807480. 00000100:00000200:2.0:1773941498.849789:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849790:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e688. 00000400:00000010:2.0:1773941498.849791:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18807600. 00000100:00000200:2.0:1773941498.849792:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849793:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380ef78. 00000400:00000010:2.0:1773941498.849795:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18807e40. 00000100:00000200:2.0:1773941498.849796:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849797:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380ea50. 00000400:00000010:2.0:1773941498.849799:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18807240. 00000100:00000200:2.0:1773941498.849805:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849806:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e108. 00000400:00000010:2.0:1773941498.849808:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18807000. 00000100:00000200:2.0:1773941498.849809:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849810:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380eaa8. 00000400:00000010:2.0:1773941498.849812:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18807780. 00000100:00000200:2.0:1773941498.849813:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849814:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e160. 00000400:00000010:2.0:1773941498.849815:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18807cc0. 00000100:00000200:2.0:1773941498.849817:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849818:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380ed10. 00000400:00000010:2.0:1773941498.849819:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d188070c0. 00000100:00000200:2.0:1773941498.849820:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849822:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e580. 00000400:00000010:2.0:1773941498.849823:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18807300. 00000100:00000200:2.0:1773941498.849824:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849826:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e5d8. 00000400:00000010:2.0:1773941498.849827:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d18807840. 00000100:00000200:2.0:1773941498.849828:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849829:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e7e8. 00000400:00000010:2.0:1773941498.849831:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d21c4c300. 00000100:00000200:2.0:1773941498.849833:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849834:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e840. 00000400:00000010:2.0:1773941498.849836:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d21c4c6c0. 00000100:00000200:2.0:1773941498.849838:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849840:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e370. 00000400:00000010:2.0:1773941498.849842:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d21c4c000. 00000100:00000200:2.0:1773941498.849844:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849846:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e790. 00000400:00000010:2.0:1773941498.849848:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d21c4c780. 00000100:00000200:2.0:1773941498.849850:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849852:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e420. 00000400:00000010:2.0:1773941498.849855:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d21c4c3c0. 00000100:00000200:2.0:1773941498.849857:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849859:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380ebb0. 00000400:00000010:2.0:1773941498.849867:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d21c4c840. 00000100:00000200:2.0:1773941498.849869:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849871:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e210. 00000400:00000010:2.0:1773941498.849873:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d21c4ca80. 00000100:00000200:2.0:1773941498.849874:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849875:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e528. 00000400:00000010:2.0:1773941498.849877:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d21c4cf00. 00000100:00000200:2.0:1773941498.849878:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849879:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e268. 00000400:00000010:2.0:1773941498.849881:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d21c4c180. 00000100:00000200:2.0:1773941498.849882:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849883:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e9a0. 00000400:00000010:2.0:1773941498.849884:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d21c4c900. 00000100:00000200:2.0:1773941498.849885:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849887:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e2c0. 00000400:00000010:2.0:1773941498.849888:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d21c4cc00. 00000100:00000200:2.0:1773941498.849889:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849890:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380eec8. 00000400:00000010:2.0:1773941498.849891:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d21c4c240. 00000100:00000200:2.0:1773941498.849893:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849894:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0380e058. 00000400:00000010:2.0:1773941498.849895:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d21c4c540. 00000100:00000200:2.0:1773941498.849896:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849898:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379000. 00000400:00000010:2.0:1773941498.849899:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d21c4ce40. 00000100:00000200:2.0:1773941498.849900:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849901:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379cb8. 00000400:00000010:2.0:1773941498.849903:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d21c4cd80. 00000100:00000200:2.0:1773941498.849904:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849905:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379b58. 00000400:00000010:2.0:1773941498.849906:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d21c4c600. 00000100:00000200:2.0:1773941498.849907:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849909:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379f20. 00000400:00000010:2.0:1773941498.849910:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d21c4c480. 00000100:00000200:2.0:1773941498.849911:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849915:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379d68. 00000400:00000010:2.0:1773941498.849917:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d21c4c0c0. 00000100:00000200:2.0:1773941498.849918:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849920:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203798f0. 00000400:00000010:2.0:1773941498.849921:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d21c4ccc0. 00000100:00000200:2.0:1773941498.849922:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849923:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203793c8. 00000400:00000010:2.0:1773941498.849925:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d09350a80. 00000100:00000200:2.0:1773941498.849926:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849927:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379dc0. 00000400:00000010:2.0:1773941498.849928:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d09350600. 00000100:00000200:2.0:1773941498.849930:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849931:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203790b0. 00000400:00000010:2.0:1773941498.849932:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d09350000. 00000100:00000200:2.0:1773941498.849933:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849934:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379738. 00000400:00000010:2.0:1773941498.849936:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d09350900. 00000100:00000200:2.0:1773941498.849937:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849938:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379478. 00000400:00000010:2.0:1773941498.849939:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d09350e40. 00000100:00000200:2.0:1773941498.849940:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849941:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379688. 00000400:00000010:2.0:1773941498.849942:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d09350d80. 00000100:00000200:2.0:1773941498.849943:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849944:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379f78. 00000400:00000010:2.0:1773941498.849946:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d09350c00. 00000100:00000200:2.0:1773941498.849947:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849948:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379a50. 00000400:00000010:2.0:1773941498.849949:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d09350300. 00000100:00000200:2.0:1773941498.849950:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849951:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379108. 00000400:00000010:2.0:1773941498.849953:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d093500c0. 00000100:00000200:2.0:1773941498.849954:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849955:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379aa8. 00000400:00000010:2.0:1773941498.849956:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d093503c0. 00000100:00000200:2.0:1773941498.849979:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849981:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379160. 00000400:00000010:2.0:1773941498.849984:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d09350f00. 00000100:00000200:2.0:1773941498.849986:0:14702:0:(niobuf.c:1190:ptlrpc_register_rqbd()) ldlm_canceld: registering portal 17 00000400:00000010:2.0:1773941498.849988:0:14702:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379d10. 00000400:00000010:2.0:1773941498.849990:0:14702:0:(lib-md.c:137:lnet_md_build()) alloc '(lmd)': 152 at ffff988d09350240. 00000100:00000010:2.0:1773941498.849993:0:14702:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff988d0544f000. 00000100:00000200:2.0:1773941498.850003:0:14702:0:(service.c:3086:ptlrpc_main()) service thread 0 (#1) started 00000100:00000001:0.0:1773941498.850014:0:14694:0:(service.c:3528:ptlrpc_start_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.850018:0:14694:0:(service.c:3433:ptlrpc_start_thread()) Process entered 00000100:00100000:0.0:1773941498.850020:0:14694:0:(service.c:3441:ptlrpc_start_thread()) ldlm_canceld[1] started 1 min 3 max 34 00000100:00000010:0.0:1773941498.850026:0:14694:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff988d10400200. 00000100:00100000:0.0:1773941498.850029:0:14694:0:(service.c:3500:ptlrpc_start_thread()) starting thread 'ldlm_cn01_001' 00000100:00000001:0.0:1773941498.850141:0:14703:0:(service.c:2998:ptlrpc_main()) Process entered 00000100:00000010:2.0:1773941498.850176:0:14703:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff988d05437640. 00000020:00000010:2.0:1773941498.850179:0:14703:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff988d054375c0. 00000020:00000010:2.0:1773941498.850182:0:14703:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d0350ec00. 00000020:00000010:2.0:1773941498.850184:0:14703:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d0350f000. 00000020:00000010:2.0:1773941498.850186:0:14703:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d08bbbb40. 00000040:00000010:2.0:1773941498.850188:0:14703:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d0350f800. 00000020:00000010:2.0:1773941498.850190:0:14703:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d0544c000. 00000020:00000010:2.0:1773941498.850193:0:14703:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff988d05400000. 00000020:00000010:2.0:1773941498.850195:0:14703:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d0350f600. 80000000:00000010:2.0:1773941498.850196:0:14703:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d08bbb6c0. 40000000:00000010:2.0:1773941498.850198:0:14703:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff988d05437e00. 00008000:00000010:2.0:1773941498.850200:0:14703:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d08326350. 00100000:00000010:2.0:1773941498.850202:0:14703:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff988d0544b000. 00040000:00000010:2.0:1773941498.850205:0:14703:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d0a0db100. 00040000:00000010:2.0:1773941498.850207:0:14703:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff988d0544c800. 00040000:00000010:2.0:1773941498.850209:0:14703:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d0350e200. 00000004:00000010:2.0:1773941498.850211:0:14703:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff988d0544d800. 00000004:00000010:2.0:1773941498.850230:0:14703:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff988d08afe000. 00000004:00000010:2.0:1773941498.850232:0:14703:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff988d0544e800. 00002000:00000010:2.0:1773941498.850234:0:14703:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff988d035a5400. 00000004:00000010:2.0:1773941498.850236:0:14703:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff988d035a6000. 00000100:00000010:2.0:1773941498.850238:0:14703:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff988d0544d000. 00000100:00000200:2.0:1773941498.850247:0:14703:0:(service.c:3086:ptlrpc_main()) service thread 1 (#2) started 00000100:00000001:1.0:1773941498.850259:0:14694:0:(service.c:3528:ptlrpc_start_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941498.850263:0:14694:0:(service.c:3433:ptlrpc_start_thread()) Process entered 00000100:00100000:1.0:1773941498.850264:0:14694:0:(service.c:3441:ptlrpc_start_thread()) ldlm_canceld[1] started 2 min 3 max 34 00000100:00000010:1.0:1773941498.850270:0:14694:0:(service.c:3452:ptlrpc_start_thread()) kmalloced '(thread)': 328 at ffff988d08a4e000. 00000100:00100000:1.0:1773941498.850273:0:14694:0:(service.c:3500:ptlrpc_start_thread()) starting thread 'ldlm_cn01_002' 00000100:00000001:1.0:1773941498.850405:0:14704:0:(service.c:2998:ptlrpc_main()) Process entered 00000100:00000010:2.0:1773941498.851865:0:14704:0:(service.c:3025:ptlrpc_main()) kmalloced '(env)': 56 at ffff988d05437f80. 00000020:00000010:2.0:1773941498.851868:0:14704:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff988d05437a40. 00000020:00000010:2.0:1773941498.851873:0:14704:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d07ebbc00. 00000020:00000010:2.0:1773941498.851882:0:14704:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d0381f000. 00000020:00000010:2.0:1773941498.851885:0:14704:0:(cl_object.c:1137:cl_key_init()) kmalloced '(value)': 72 at ffff988d08bbb5a0. 00000040:00000010:2.0:1773941498.851889:0:14704:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d0381ec00. 00000400:00000200:3.0:1773941498.851891:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000020:00000010:2.0:1773941498.851895:0:14704:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d05448000. 00000400:00000010:3.0:1773941498.851902:0:2399:0:(peer.c:3405:lnet_peer_merge_data()) kfreed 'curnis': 40 at ffff988d03ec75c0. 00000020:00000010:2.0:1773941498.851902:0:14704:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff988d05403c60. 00000400:00000010:3.0:1773941498.851909:0:2399:0:(peer.c:3406:lnet_peer_merge_data()) kfreed 'addnis': 48 at ffff988d03ec7080. 00000020:00000010:2.0:1773941498.851910:0:14704:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d0381fe00. 00000400:00000010:3.0:1773941498.851912:0:2399:0:(peer.c:3407:lnet_peer_merge_data()) kfreed 'delnis': 40 at ffff988d03ec7600. 80000000:00000010:2.0:1773941498.851915:0:14704:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d08bbb3c0. 00000400:00000010:3.0:1773941498.851917:0:2399:0:(api-ni.c:1839:lnet_ping_buffer_free()) kfreed 'pbuf': 281 at ffff988d1817d600. 00000800:00000010:1.0:1773941498.851921:0:2389:0:(socklnd.c:886:ksocknal_create_conn()) alloc '(conn)': 4712 at ffff988d18bfe000. 40000000:00000010:2.0:1773941498.851922:0:14704:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff988d05437f40. 00000400:00000200:3.0:1773941498.851926:0:2399:0:(peer.c:3410:lnet_peer_merge_data()) peer 192.168.204.114@tcp (ffff988d03357600): 0 00000800:00000010:1.0:1773941498.851929:0:2389:0:(socklnd.c:913:ksocknal_create_conn()) alloc '(hello)': 144 at ffff988d08aaf0c0. 00008000:00000010:2.0:1773941498.851930:0:14704:0:(echo_client.c:413:echo_thread_key_init()) slab-alloced 'info': 904 at ffff988d083251a8. 00000400:00000200:3.0:1773941498.851931:0:2399:0:(peer.c:3714:lnet_peer_data_present()) peer 192.168.204.114@tcp(ffff988d03357600): 0. state = 0x100141 00000800:00000010:1.0:1773941498.851935:0:2389:0:(socklnd_proto.c:573:ksocknal_send_hello_v2()) alloc '(hello4)': 56 at ffff988d04d67e40. 00100000:00000010:2.0:1773941498.851939:0:14704:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff988d0544a800. 00000400:00000200:3.0:1773941498.851941:0:2399:0:(peer.c:4118:lnet_peer_discovery()) peer 192.168.204.114@tcp(ffff988d03357600) state 0x100141 rc 1 00000800:00000010:1.0:1773941498.851945:0:2389:0:(socklnd_proto.c:613:ksocknal_send_hello_v2()) kfreed 'hello4': 56 at ffff988d04d67e40. 00040000:00000010:2.0:1773941498.851947:0:14704:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d0a0db200. 00040000:00000010:2.0:1773941498.851953:0:14704:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff988d05448800. 00040000:00000010:2.0:1773941498.851956:0:14704:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d0381fa00. 00000004:00000010:2.0:1773941498.851989:0:14704:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff988d0544a000. 00000004:00000010:2.0:1773941498.851994:0:14704:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff988d08af8000. 00000004:00000010:2.0:1773941498.851999:0:14704:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff988d09304800. 00002000:00000010:2.0:1773941498.852003:0:14704:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff988d035a6c00. 00000004:00000010:2.0:1773941498.852005:0:14704:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff988d035a7000. 00000100:00000010:2.0:1773941498.852008:0:14704:0:(service.c:3052:ptlrpc_main()) kmalloced '(rs)': 2048 at ffff988d09301000. 00000100:00000200:2.0:1773941498.852022:0:14704:0:(service.c:3086:ptlrpc_main()) service thread 2 (#3) started 00000400:00000200:3.0:1773941498.852061:0:2399:0:(peer.c:4100:lnet_peer_discovery()) peer 192.168.204.114@tcp(ffff988d03357600) state 0x100141 00000400:00000010:3.0:1773941498.852068:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac198. 00000400:00000010:3.0:1773941498.852074:0:2399:0:(lib-lnet.h:478:lnet_rspt_alloc()) alloc 'rspt': 56 at ffff988d09a24380. 00000400:00000200:3.0:1773941498.852079:0:2399:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.124@tcp 00000400:00000200:3.0:1773941498.852086:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000100:00000001:0.0:1773941498.852088:0:14694:0:(service.c:3528:ptlrpc_start_thread()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941498.852092:0:2399:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source Specified: 192.168.204.14@tcp to MR: 192.168.204.124@tcp local destination 00000100:00000001:0.0:1773941498.852093:0:14694:0:(service.c:3418:ptlrpc_start_threads()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.852097:0:14694:0:(service.c:857:ptlrpc_register_service()) Process leaving (rc=18446630330195155968 : -113743514395648 : ffff988d063a9c00) 00000400:00000200:3.0:1773941498.852102:0:2399:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [112:112:256:4294967295] 192.168.204.124@tcp LPNI seq info [2:2:8:4294967295] 00010000:00000010:0.0:1773941498.852105:0:14694:0:(ldlm_lockd.c:3466:ldlm_setup()) kmalloced '(blp)': 168 at ffff988d02eb6180. 00000400:00000200:3.0:1773941498.852109:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:3.0:1773941498.852114:0:2399:0:(lib-move.c:2009:lnet_handle_send()) rspt_next_hop_nid = 192.168.204.124@tcp 00000400:00000200:3.0:1773941498.852119:0:2399:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:192.168.204.14@tcp) -> 192.168.204.124@tcp(192.168.204.124@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:3.0:1773941498.852124:0:2399:0:(socklnd_cb.c:1001:ksocknal_send()) sending 48 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:3.0:1773941498.852128:0:2399:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d21731700. 00000800:00000200:3.0:1773941498.852133:0:2399:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (4) 00000800:00000200:3.0:1773941498.852138:0:2399:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:3.0:1773941498.852141:0:2399:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d21731700 type 1, nob 144 niov 1 nkiov 1 00000400:00000200:3.0:1773941498.852154:0:2399:0:(peer.c:3922:lnet_peer_send_push()) peer 192.168.204.114@tcp 00000400:00000200:3.0:1773941498.852157:0:2399:0:(peer.c:4118:lnet_peer_discovery()) peer 192.168.204.114@tcp(ffff988d03357600) state 0x100541 rc 0 00000800:00000200:0.0:1773941498.852208:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941498.852211:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d21731700. 00000400:00000200:0.0:1773941498.852223:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941498.852228:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941498.852231:0:2393:0:(peer.c:3081:lnet_discovery_event_handler()) Received event: 5 00000400:00000200:0.0:1773941498.852233:0:2393:0:(peer.c:3033:lnet_discovery_event_send()) Push Send to 192.168.204.124@tcp: 0 00010000:00000001:3.0:1773941498.852273:0:14705:0:(ldlm_lockd.c:2959:ldlm_bl_thread_main()) Process entered 00010000:00000010:3.0:1773941498.852380:0:14705:0:(ldlm_lockd.c:2961:ldlm_bl_thread_main()) kmalloced '(env)': 56 at ffff988d03ec7600. 00000020:00000010:3.0:1773941498.852383:0:14705:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d1a204a00. 00000020:00000010:3.0:1773941498.852387:0:14705:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d1817d600. 00000020:00000010:3.0:1773941498.852389:0:14705:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d0835b800. 00000020:00000010:3.0:1773941498.852392:0:14705:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff988d03340f18. 00000020:00000010:3.0:1773941498.852395:0:14705:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d04f8ae00. 80000000:00000010:3.0:1773941498.852402:0:14705:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d111f7000. 40000000:00000010:3.0:1773941498.852404:0:14705:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff988d03ec7080. 00100000:00000010:3.0:1773941498.852407:0:14705:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff988d0835a000. 00040000:00000010:3.0:1773941498.852409:0:14705:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d21731c00. 00040000:00000010:3.0:1773941498.852412:0:14705:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d02d18800. 00002000:00000010:3.0:1773941498.852455:0:14705:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff988d03521000. 00000020:00000010:3.0:1773941498.852460:0:14705:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff988d03ec75c0. 00000020:00000001:3.0:1773941498.852475:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941498.852478:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941498.852482:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1773941498.852484:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941498.852486:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941498.852487:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1773941498.852489:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941498.852490:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941498.852491:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1773941498.852623:0:14706:0:(ldlm_lockd.c:2959:ldlm_bl_thread_main()) Process entered 00010000:00000010:3.0:1773941498.852626:0:14706:0:(ldlm_lockd.c:2961:ldlm_bl_thread_main()) kmalloced '(env)': 56 at ffff988d03ec7d40. 00000020:00000010:3.0:1773941498.852628:0:14706:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d02d19600. 00000020:00000010:3.0:1773941498.852631:0:14706:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d02d19400. 00000020:00000010:3.0:1773941498.852634:0:14706:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d08359000. 00000020:00000010:3.0:1773941498.852638:0:14706:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff988d03342840. 00000020:00000010:3.0:1773941498.852641:0:14706:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d02d18400. 80000000:00000010:3.0:1773941498.852709:0:14706:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d111f7ea0. 40000000:00000010:3.0:1773941498.852713:0:14706:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff988d03ec7800. 00100000:00000010:3.0:1773941498.852716:0:14706:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff988d0835c000. 00040000:00000010:3.0:1773941498.852718:0:14706:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d21731700. 00040000:00000010:3.0:1773941498.852720:0:14706:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d02d18c00. 00002000:00000010:3.0:1773941498.852723:0:14706:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff988d03521800. 00000020:00000010:3.0:1773941498.852725:0:14706:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff988d03ec79c0. 00000020:00000001:3.0:1773941498.852733:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941498.852734:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941498.852736:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1773941498.852737:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941498.852743:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941498.852744:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1773941498.852745:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941498.852746:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941498.852748:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:3.0:1773941498.852891:0:14707:0:(ldlm_lockd.c:159:expired_lock_main()) Process entered 00000020:00000010:3.0:1773941498.852894:0:14707:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d02d18600. 00000020:00000010:3.0:1773941498.852897:0:14707:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d02d18a00. 00000040:00000010:3.0:1773941498.852899:0:14707:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d02d18e00. 00000020:00000010:3.0:1773941498.852902:0:14707:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d0835f000. 00000020:00000010:3.0:1773941498.852904:0:14707:0:(tgt_main.c:719:tgt_key_init()) slab-alloced 'thread': 1288 at ffff988d03345a90. 00000020:00000010:3.0:1773941498.852907:0:14707:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d02d19a00. 80000000:00000010:3.0:1773941498.852909:0:14707:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d111f7120. 40000000:00000010:3.0:1773941498.852911:0:14707:0:(fid_handler.c:473:seq_key_init()) kmalloced '(value)': 48 at ffff988d03ec77c0. 00100000:00000010:3.0:1773941498.852914:0:14707:0:(lfsck_lib.c:33:lfsck_key_init()) kmalloced '(value)': 1800 at ffff988d08994800. 00040000:00000010:3.0:1773941498.852917:0:14707:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d0a21c500. 00040000:00000010:3.0:1773941498.852919:0:14707:0:(qmt_dev.c:344:qmt_key_init()) kmalloced '(value)': 1256 at ffff988d08992000. 00040000:00000010:3.0:1773941498.852921:0:14707:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d0642d200. 00000004:00000010:3.0:1773941498.852924:0:14707:0:(mdd_device.c:2406:mdd_key_init()) kmalloced '(value)': 1872 at ffff988d08990800. 00000004:00000010:3.0:1773941498.852928:0:14707:0:(mdt_handler.c:8397:mdt_key_init()) kmalloced '(value)': 2568 at ffff988d02d6b000. 00000004:00000010:3.0:1773941498.852931:0:14707:0:(lod_dev.c:2340:lod_key_init()) kmalloced '(value)': 1224 at ffff988d08990000. 00002000:00000010:3.0:1773941498.852935:0:14707:0:(ofd_dev.c:3000:ofd_key_init()) kmalloced '(value)': 816 at ffff988d03520400. 00000004:00000010:3.0:1773941498.852937:0:14707:0:(osp_dev.c:1819:osp_key_init()) kmalloced '(value)': 616 at ffff988d03523400. 00000020:00000010:3.0:1773941498.852939:0:14707:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff988d03ec71c0. 00010000:00000001:3.0:1773941498.852948:0:14707:0:(ldlm_lockd.c:142:have_expired_locks()) Process entered 00010000:00000001:3.0:1773941498.852949:0:14707:0:(ldlm_lockd.c:147:have_expired_locks()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941498.852951:0:14707:0:(ldlm_lockd.c:142:have_expired_locks()) Process entered 00010000:00000001:3.0:1773941498.852952:0:14707:0:(ldlm_lockd.c:147:have_expired_locks()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941498.852989:0:14694:0:(ldlm_pool.c:1348:ldlm_pools_init()) Process entered 00000001:00000010:0.0:1773941498.852996:0:14694:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at ffff988d06580a80. 00000001:00000010:0.0:1773941498.853004:0:14694:0:(shrinker_debug.c:228:ll_shrinker_alloc()) alloc '(s)': 88 at ffff988d06580840. 00010000:00000001:0.0:1773941498.853011:0:14694:0:(ldlm_pool.c:1376:ldlm_pools_init()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941498.853024:0:14694:0:(ldlm_lockd.c:3518:ldlm_setup()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941498.853027:0:14694:0:(ldlm_lockd.c:3039:ldlm_get_ref()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1773941498.853033:0:14694:0:(genops.c:1283:class_new_import()) kmalloced '(imp)': 1712 at ffff988d20336000. 00000020:00000040:0.0:1773941498.853042:0:14694:0:(obd_config.c:1030:class_incref()) incref MGC192.168.204.114@tcp (ffff988d2039cd28) now 3 - import 00000020:00000001:0.0:1773941498.853048:0:14694:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941498.853051:0:14694:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=1 obd=MGC192.168.204.114@tcp 00000020:00000001:0.0:1773941498.853053:0:14694:0:(genops.c:1249:class_import_put()) Process leaving 00010000:00000001:0.0:1773941498.853057:0:14694:0:(ldlm_lib.c:49:import_set_conn()) Process entered 00000100:00000200:0.0:1773941498.853069:0:14694:0:(events.c:560:ptlrpc_uuid_to_peer()) 192.168.204.114@tcp->12345-192.168.204.114@tcp 00000100:00000001:0.0:1773941498.853072:0:14694:0:(connection.c:104:ptlrpc_connection_get()) Process entered 00000400:00000200:0.0:1773941498.853076:0:14694:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941498.853083:0:14694:0:(peer.c:2405:lnet_peer_queue_for_discovery()) Queue peer 192.168.204.114@tcp: -114 00000400:00000200:0.0:1773941498.853086:0:14694:0:(peer.c:2727:lnet_discover_peer_locked()) Discovery attempt # 1 00000400:00000200:0.0:1773941498.853089:0:14694:0:(peer.c:2767:lnet_discover_peer_locked()) non-blocking discovery 00000400:00000200:0.0:1773941498.853092:0:14694:0:(peer.c:2776:lnet_discover_peer_locked()) peer 192.168.204.114@tcp NID 192.168.204.114@tcp: 0. pending discovery 00000400:00000200:0.0:1773941498.853097:0:14694:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941498.853100:0:14694:0:(peer.c:1471:LNetPrimaryNID()) NID 192.168.204.114@tcp primary NID 192.168.204.114@tcp rc 0 00000100:00000001:0.0:1773941498.853104:0:14694:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941498.853107:0:14694:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 1 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941498.853110:0:14694:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000100:00000001:0.0:1773941498.853121:0:14694:0:(connection.c:110:ptlrpc_connection_get()) Process leaving via out (rc=18446630330227265856 : -113743482285760 : 0xffff988d08249140) 00000100:00000040:0.0:1773941498.853129:0:14694:0:(connection.c:149:ptlrpc_connection_get()) conn=ffff988d08249140 refcount 1 to 192.168.204.114@tcp 00000100:00000040:0.0:1773941498.853132:0:14694:0:(client.c:137:ptlrpc_uuid_to_connection()) 192.168.204.114@tcp -> ffff988d08249140 00010000:00000010:0.0:1773941498.853137:0:14694:0:(ldlm_lib.c:70:import_set_conn()) kmalloced '(imp_conn)': 88 at ffff988d06580720. 00010000:00080000:0.0:1773941498.853140:0:14694:0:(ldlm_lib.c:102:import_set_conn()) imp ffff988d20336000@MGC192.168.204.114@tcp: add connection 192.168.204.114@tcp at head 00010000:00000001:0.0:1773941498.853144:0:14694:0:(ldlm_lib.c:109:import_set_conn()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941498.853147:0:14694:0:(ldlm_resource.c:938:ldlm_namespace_new()) Process entered 00010000:00000001:0.0:1773941498.853149:0:14694:0:(ldlm_lockd.c:3030:ldlm_get_ref()) Process entered 00010000:00000001:0.0:1773941498.853151:0:14694:0:(ldlm_lockd.c:3039:ldlm_get_ref()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:0.0:1773941498.853155:0:14694:0:(ldlm_resource.c:955:ldlm_namespace_new()) kmalloced '(ns)': 848 at ffff988d1125a400. 00000001:00000001:0.0:1773941498.853158:0:14694:0:(hash.c:1038:cfs_hash_create()) Process entered 00000001:00000010:0.0:1773941498.853163:0:14694:0:(hash.c:1065:cfs_hash_create()) alloc '(hs)': 280 at ffff988d10401400. 00000001:00000010:0.0:1773941498.853168:0:14694:0:(hash.c:933:cfs_hash_buckets_realloc()) alloc '(new_bkts)': 16 at ffff988d065cd9c0. 00000001:00000010:0.0:1773941498.853172:0:14694:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 304 at ffff988d10400a00. 00000001:00000010:0.0:1773941498.853176:0:14694:0:(hash.c:946:cfs_hash_buckets_realloc()) alloc '(new_bkts[i])': 304 at ffff988d10401a00. 00000020:00000010:0.0:1773941498.853239:0:14694:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff988d02ce4400. 00000020:00000010:0.0:1773941498.853247:0:14694:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 96 at ffff988d065807e0. 00010000:00000001:0.0:1773941498.853266:0:14694:0:(ldlm_pool.c:890:ldlm_pool_init()) Process entered 00010000:00000001:0.0:1773941498.853284:0:14694:0:(ldlm_pool.c:809:ldlm_pool_debugfs_init()) Process entered 00000020:00000010:0.0:1773941498.853296:0:14694:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff988d02ce5400. 00000020:00000010:0.0:1773941498.853298:0:14694:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 352 at ffff988d02cbde00. 00010000:00000001:0.0:1773941498.853308:0:14694:0:(ldlm_pool.c:864:ldlm_pool_debugfs_init()) Process leaving 00010000:00010000:0.0:1773941498.853310:0:14694:0:(ldlm_pool.c:924:ldlm_pool_init()) Lock pool ldlm-pool-MGC192.168.204.114@tcp-0 is initialized 00010000:00000001:0.0:1773941498.853311:0:14694:0:(ldlm_pool.c:926:ldlm_pool_init()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941498.853313:0:14694:0:(ldlm_resource.c:1046:ldlm_namespace_new()) Process leaving (rc=18446630330378331136 : -113743331220480 : ffff988d1125a400) 00010000:00000001:0.0:1773941498.853315:0:14694:0:(ldlm_lib.c:605:client_obd_setup()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941498.853318:0:14694:0:(mgc_request.c:751:mgc_llog_init()) Process entered 00000040:00000001:0.0:1773941498.853320:0:14694:0:(llog_obd.c:133:llog_setup()) Process entered 00000040:00000010:0.0:1773941498.853322:0:14694:0:(llog_obd.c:26:llog_new_ctxt()) kmalloced '(ctxt)': 168 at ffff988d02eb6540. 00000020:00000040:0.0:1773941498.853324:0:14694:0:(genops.c:887:class_export_get()) GET export ffff988d111b5800 refcount=2 00000040:01000000:0.0:1773941498.853326:0:14694:0:(llog_obd.c:195:llog_setup()) obd MGC192.168.204.114@tcp ctxt 1 is initialized 00000040:00000001:0.0:1773941498.853328:0:14694:0:(llog_obd.c:199:llog_setup()) Process leaving (rc=0 : 0 : 0) 10000000:00000040:0.0:1773941498.853329:0:14694:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 00000040:00000001:0.0:1773941498.853332:0:14694:0:(llog_net.c:31:llog_initiator_connect()) Process entered 00000020:00000040:0.0:1773941498.853334:0:14694:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=2 obd=MGC192.168.204.114@tcp 00000040:00000001:0.0:1773941498.853336:0:14694:0:(llog_net.c:44:llog_initiator_connect()) Process leaving (rc=0 : 0 : 0) 10000000:00000040:0.0:1773941498.853337:0:14694:0:(lustre_log.h:400:llog_ctxt_put()) PUTting ctxt ffff988d02eb6540 : new refcount 1 10000000:00000001:0.0:1773941498.853339:0:14694:0:(mgc_request.c:767:mgc_llog_init()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941498.853426:0:14694:0:(lprocfs_status.c:1339:lprocfs_obd_setup()) Process leaving via already_registered (rc=0 : 0 : 0x0) 10000000:00000001:2.0:1773941498.853580:0:14708:0:(mgc_request.c:629:mgc_requeue_thread()) Process entered 10000000:01000000:2.0:1773941498.853583:0:14708:0:(mgc_request.c:631:mgc_requeue_thread()) Starting requeue thread 10000000:00000001:0.0:1773941498.853603:0:14694:0:(mgc_request.c:896:mgc_device_alloc()) Process leaving (rc=18446630330367945536 : -113743341606080 : ffff988d10872b40) 00000020:00000010:0.0:1773941498.853610:0:14694:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d03357800. 00000040:00000010:0.0:1773941498.853615:0:14694:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d03357e00. 00000020:00000010:0.0:1773941498.853618:0:14694:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d111b0800. 00000020:00000010:0.0:1773941498.853622:0:14694:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d03356600. 00040000:00000010:0.0:1773941498.853626:0:14694:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d04c1a400. 00040000:00000010:0.0:1773941498.853631:0:14694:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d03356400. 00000020:00000010:0.0:1773941498.853634:0:14694:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d03356000. 00000020:00000010:0.0:1773941498.853639:0:14694:0:(lu_ucred.c:27:lu_ucred_key_fini()) kfreed 'info': 128 at ffff988d181fef80. 00000020:00000010:0.0:1773941498.853643:0:14694:0:(tgt_main.c:768:tgt_ses_key_fini()) slab-freed '(session)': 200 at ffff988d06522258. 00008000:00000010:0.0:1773941498.853647:0:14694:0:(echo_client.c:449:echo_session_key_fini()) slab-freed '(session)': 8 at ffff988d065239c8. 00000020:00000010:0.0:1773941498.853656:0:14694:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d03356200. 00000020:00000001:0.0:1773941498.853659:0:14694:0:(obd_class.h:593:obd_setup()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1773941498.853662:0:14694:0:(obd_config.c:1030:class_incref()) incref MGC192.168.204.114@tcp (ffff988d2039cd28) now 4 - setup 00000020:00000080:0.0:1773941498.853665:0:14694:0:(obd_config.c:861:class_setup()) finished setup of obd MGC192.168.204.114@tcp (uuid c1c1ff21-5f68-4bf6-bdc4-258644b24465) 00000020:00000001:0.0:1773941498.853667:0:14694:0:(obd_config.c:863:class_setup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941498.853669:0:14694:0:(obd_config.c:1651:class_process_config()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1773941498.853673:0:14694:0:(obd_mount.c:168:do_lcfg()) kfreed 'lcfg': 104 at ffff988d181fe500. 00000020:00000001:0.0:1773941498.853680:0:14694:0:(obd_class.h:528:obd_set_info_async()) Process entered 10000000:00000001:0.0:1773941498.853682:0:14694:0:(mgc_request.c:1055:mgc_set_info_async()) Process entered 10000000:00000001:0.0:1773941498.853687:0:14694:0:(mgc_request.c:1125:mgc_set_info_async()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941498.853689:0:14694:0:(obd_class.h:545:obd_set_info_async()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941498.853691:0:14694:0:(obd_class.h:852:obd_connect()) Process entered 00010000:00000001:0.0:1773941498.853693:0:14694:0:(ldlm_lib.c:670:client_connect_import()) Process entered 00000020:00000001:0.0:1773941498.853695:0:14694:0:(genops.c:1394:class_connect()) Process entered 00000020:00000001:0.0:1773941498.853697:0:14694:0:(genops.c:950:__class_new_export()) Process entered 00000020:00000010:0.0:1773941498.853700:0:14694:0:(genops.c:952:__class_new_export()) kmalloced '(export)': 1208 at ffff988d111b0800. 00000020:00000001:0.0:1773941498.853703:0:14694:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:0.0:1773941498.853705:0:14694:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d111b0800 with handle 0x8e86767622e28d4 to hash 00000020:00000001:0.0:1773941498.853708:0:14694:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00000020:00000001:0.0:1773941498.853710:0:14694:0:(obd_class.h:1008:obd_init_export()) Process entered 00000020:00000001:0.0:1773941498.853712:0:14694:0:(obd_class.h:1012:obd_init_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1773941498.853714:0:14694:0:(obd_config.c:1030:class_incref()) incref MGC192.168.204.114@tcp (ffff988d2039cd28) now 5 - export 00000020:00000001:0.0:1773941498.853717:0:14694:0:(genops.c:1020:__class_new_export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000040:0.0:1773941498.853720:0:14694:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 1 00000020:00000080:0.0:1773941498.853723:0:14694:0:(genops.c:1404:class_connect()) connect: client c1c1ff21-5f68-4bf6-bdc4-258644b24465, cookie 0x8e86767622e28d4 00000020:00000001:0.0:1773941498.853725:0:14694:0:(genops.c:1405:class_connect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941498.853727:0:14694:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941498.853728:0:14694:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:0.0:1773941498.853730:0:14694:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941498.853733:0:14694:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=2 00000020:00000001:0.0:1773941498.853735:0:14694:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:0.0:1773941498.853737:0:14694:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000001:0.0:1773941498.853741:0:14694:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1773941498.853745:0:14694:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1773941498.853755:0:14694:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 0 00000400:00000200:0.0:1773941498.853760:0:14694:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941498.853765:0:14694:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is not discovered: rc = -114 00000100:00080000:0.0:1773941498.853768:0:14694:0:(import.c:545:import_select_connection()) MGC192.168.204.114@tcp: skip NID 192.168.204.114@tcp as not ready: rc = -114 00000100:00000001:0.0:1773941498.853771:0:14694:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941498.853773:0:14694:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941498.853776:0:14694:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:0.0:1773941498.853779:0:14694:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941498.853780:0:14694:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:0.0:1773941498.853781:0:14694:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941498.853783:0:14694:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:0.0:1773941498.853785:0:14694:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:0.0:1773941498.853787:0:14694:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000001:0.0:1773941498.853789:0:14694:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941498.853792:0:14694:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941498.853794:0:14694:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:0.0:1773941498.853796:0:14694:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:0.0:1773941498.853800:0:14694:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:0.0:1773941498.853803:0:14694:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941498.853806:0:14694:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1773941498.853808:0:14694:0:(sec.c:1432:sptlrpc_sec_create()) Process entered 02000000:08000000:0.0:1773941498.853811:0:14694:0:(sec.c:1450:sptlrpc_sec_create()) mgc MGC192.168.204.114@tcp: select security flavor null 02000000:00000001:0.0:1773941498.853814:0:14694:0:(sec.c:1471:sptlrpc_sec_create()) Process leaving (rc=18446744072651120288 : -1058431328 : ffffffffc0e99ea0) 02000000:00000001:0.0:1773941498.853818:0:14694:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.853820:0:14694:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:0.0:1773941498.853823:0:14694:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1773941498.853828:0:14694:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10836a00. 00000020:00000040:0.0:1773941498.853831:0:14694:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 02000000:00000001:0.0:1773941498.853834:0:14694:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1773941498.853836:0:14694:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.853839:0:14694:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941498.853842:0:14694:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941498.853850:0:14694:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941498.853857:0:14694:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d02ce6c00. 00000100:00000001:0.0:1773941498.853864:0:14694:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941498.853874:0:14694:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d10836a00 x1860112291476736/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1773941498.853885:0:14694:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941498.853889:0:14694:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941498.853895:0:14694:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f9e7d804] to pc [ptlrpcd_rcv+-1] req@ffff988d10836a00 x1860112291476736/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941498.853906:0:2389:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000100:00000001:0.0:1773941498.853923:0:14694:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.853928:0:14694:0:(pinger.c:365:ptlrpc_pinger_add_import()) Process entered 00000100:00080000:0.0:1773941498.853930:0:14694:0:(pinger.c:371:ptlrpc_pinger_add_import()) adding pingable import c1c1ff21-5f68-4bf6-bdc4-258644b24465->MGS 00000020:00000040:0.0:1773941498.853933:0:14694:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941498.853944:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000200:3.1:1773941498.853949:0:2389:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 3, ksnr_connected e ksnr_max_conns 1 00000100:00000001:2.0:1773941498.853949:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941498.853949:0:14694:0:(pinger.c:382:ptlrpc_pinger_add_import()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.853953:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00010000:00000001:0.0:1773941498.853956:0:14694:0:(ldlm_lib.c:725:client_connect_import()) Process leaving 00000100:00000040:2.0:1773941498.853994:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10836a00 x1860112291476736/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1773941498.853999:0:14694:0:(obd_class.h:868:obd_connect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.854006:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000010:0.0:1773941498.854006:0:14694:0:(obd_mount.c:561:lustre_start_mgc()) kfreed 'uuid': 40 at ffff988d10872d40. 00000100:00000001:2.0:1773941498.854008:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1773941498.854014:0:14694:0:(obd_mount.c:562:lustre_start_mgc()) kfreed 'data': 192 at ffff988d04fa2c00. 00000800:00000200:3.0:1773941498.854015:0:2389:0:(socklnd.c:1183:ksocknal_create_conn()) New conn 12345-192.168.204.124@tcp p 3.x 192.168.204.14 -> 192.168.204.124:988 incarnation:1773941238881364903 sched[0] 02000000:00000001:2.0:1773941498.854016:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000010:0.0:1773941498.854021:0:14694:0:(obd_mount.c:563:lustre_start_mgc()) kfreed 'mgcname': 23 at ffff988d06545ee0. 02000000:00000001:2.0:1773941498.854022:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000800:00000010:3.0:1773941498.854023:0:2389:0:(socklnd.c:1191:ksocknal_create_conn()) kfreed 'hello': 144 at ffff988d08aaf0c0. 00000020:00000001:0.0:1773941498.854027:0:14694:0:(obd_mount.c:565:lustre_start_mgc()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941498.854030:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000080:00000001:0.0:1773941498.854034:0:14694:0:(llite_lib.c:1392:ll_fill_super()) Process entered 00000100:00100000:2.0:1773941498.854039:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10836a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291476736:192.168.204.114@tcp:250:mount.lustre.0 00000080:00200000:0.0:1773941498.854039:0:14694:0:(llite_lib.c:1395:ll_fill_super()) VFS Op: cfg_instance lustre-client-ffff988d111b1800 (sb ffff988d111b1800) 00000080:00000010:0.0:1773941498.854046:0:14694:0:(llite_lib.c:1399:ll_fill_super()) kmalloced '(cfg)': 88 at ffff988d065809c0. 00000080:00000001:0.0:1773941498.854050:0:14694:0:(llite_lib.c:93:ll_init_sbi()) Process entered 00000080:00000010:0.0:1773941498.854054:0:14694:0:(llite_lib.c:95:ll_init_sbi()) kmalloced '(sbi)': 1376 at ffff988d111b6800. 00000100:00000001:2.0:1773941498.854060:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1773941498.854064:0:2400:0:(niobuf.c:917:ptl_send_rpc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.854067:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.854071:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.854075:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941498.854081:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941498.854082:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941498.854084:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941498.854085:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941498.854086:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941498.854088:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941498.854102:0:14694:0:(cl_page.c:1379:cl_cache_init()) Process entered 00000020:00000010:0.0:1773941498.854107:0:14694:0:(cl_page.c:1380:cl_cache_init()) kmalloced '(cache)': 152 at ffff988d02eb6e40. 00000020:00000001:0.0:1773941498.854109:0:14694:0:(cl_page.c:1396:cl_cache_init()) Process leaving (rc=18446630330139635264 : -113743569916352 : ffff988d02eb6e40) 00000080:00000010:0.0:1773941498.854113:0:14694:0:(llite_lib.c:129:ll_init_sbi()) kmalloced '(sbi->ll_foreign_symlink_prefix)': 6 at ffff988d0723df88. 00000080:00000010:0.0:1773941498.854117:0:14694:0:(llite_lib.c:136:ll_init_sbi()) kmalloced '(sbi->ll_foreign_symlink_upcall)': 5 at ffff988d0723d4b8. 00000080:00000001:0.0:1773941498.854128:0:14694:0:(llite_lib.c:240:ll_init_sbi()) Process leaving (rc=18446630330377660416 : -113743331891200 : ffff988d111b6800) 00000080:00000001:0.0:1773941498.854131:0:14694:0:(llite_lib.c:1107:ll_options()) Process entered 00000080:00000001:0.0:1773941498.854137:0:14694:0:(llite_lib.c:1109:ll_options()) Process leaving (rc=0 : 0 : 0) 00000080:01000000:0.0:1773941498.854142:0:14694:0:(llite_lib.c:1429:ll_fill_super()) llite sb uuid: b686a451-857d-4abc-b28b-7076af7cdee2 00000080:00000001:0.0:1773941498.854263:0:14694:0:(lproc_llite.c:2772:ll_debugfs_register_super()) Process entered 00000020:00000010:0.0:1773941498.854481:0:14694:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff988d02ce6400. 00000020:00000010:0.0:1773941498.854486:0:14694:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 1440 at ffff988d054a6000. 00000020:00000010:0.0:1773941498.854500:0:14694:0:(lprocfs_status.c:1445:lprocfs_stats_alloc()) alloc '(stats)': 608 at ffff988d02ce4c00. 00000020:00000010:0.0:1773941498.854503:0:14694:0:(lprocfs_status.c:1457:lprocfs_stats_alloc()) alloc '(stats->ls_cnt_header)': 576 at ffff988d02ce7c00. 00000080:00000001:0.0:1773941498.854513:0:14694:0:(lproc_llite.c:2842:ll_debugfs_register_super()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941498.854516:0:14694:0:(obd_mount.c:62:lustre_process_log()) Process entered 00000020:00000010:0.0:1773941498.854519:0:14694:0:(obd_mount.c:67:lustre_process_log()) kmalloced '(bufs)': 104 at ffff988d181fe500. 00000020:00000010:0.0:1773941498.854523:0:14694:0:(obd_mount.c:76:lustre_process_log()) kmalloced '(lcfg)': 184 at ffff988d02eb6300. 00000020:00000001:0.0:1773941498.854525:0:14694:0:(obd_class.h:681:obd_process_config()) Process entered 00000020:00000010:0.0:1773941498.854528:0:14694:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d03356000. 00000020:00000010:0.0:1773941498.854531:0:14694:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d03356400. 00000040:00000010:0.0:1773941498.854534:0:14694:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d03356600. 00000020:00000010:0.0:1773941498.854538:0:14694:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d054a7800. 00000020:00000010:0.0:1773941498.854542:0:14694:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d03357e00. 00040000:00000010:0.0:1773941498.854545:0:14694:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d0a20be00. 00040000:00000010:0.0:1773941498.854548:0:14694:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d03357800. 10000000:00000001:0.0:1773941498.854552:0:14694:0:(mgc_request.c:1932:mgc_process_config()) Process entered 10000000:00000001:0.0:1773941498.854554:0:14694:0:(mgc_request_server.c:668:mgc_process_config_server()) Process entered 10000000:01000000:0.0:1773941498.854557:0:14694:0:(mgc_request.c:1953:mgc_process_config()) parse_log lustre-client from 0 10000000:00000001:0.0:1773941498.854559:0:14694:0:(mgc_request.c:326:config_log_add()) Process entered 10000000:01000000:0.0:1773941498.854561:0:14694:0:(mgc_request.c:329:config_log_add()) add config log lustre-client-ffff988d111b1800 10000000:00000001:0.0:1773941498.854564:0:14694:0:(mgc_request.c:159:config_log_find()) Process entered 10000000:00000001:0.0:1773941498.854566:0:14694:0:(mgc_request.c:177:config_log_find()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941498.854568:0:14694:0:(mgc_request.c:191:do_config_log_add()) Process entered 10000000:01000000:0.0:1773941498.854570:0:14694:0:(mgc_request.c:194:do_config_log_add()) do adding config log lustre-sptlrpc-ffff988d2039cd28 10000000:00000010:0.0:1773941498.854574:0:14694:0:(mgc_request.c:197:do_config_log_add()) kmalloced '(cld)': 287 at ffff988d065c4800. 10000000:01000000:0.0:1773941498.854577:0:14694:0:(mgc_request.c:68:mgc_name2resid()) log lustre-sptlrpc to resid 0x65727473756c/0x0 (lustre) 00000020:00000040:0.0:1773941498.854581:0:14694:0:(genops.c:887:class_export_get()) GET export ffff988d111b5800 refcount=3 02000000:00000010:0.0:1773941498.854585:0:14694:0:(sec_config.c:546:sptlrpc_conf_get()) kmalloced '(conf)': 120 at ffff988d181fef80. 02000000:08000000:0.0:1773941498.854588:0:14694:0:(sec_config.c:557:sptlrpc_conf_get()) create sptlrpc conf lustre 10000000:00000001:0.0:1773941498.854591:0:14694:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:0.0:1773941498.854597:0:14694:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:0.0:1773941498.854600:0:14694:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1773941498.854602:0:14694:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:0.0:1773941498.854606:0:14694:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10835180. 00000020:00000040:0.0:1773941498.854609:0:14694:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941498.854611:0:14694:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941498.854614:0:14694:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941498.854616:0:14694:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941498.854619:0:14694:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065c5a00. 00000100:00000001:0.0:1773941498.854623:0:14694:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941498.854628:0:14694:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1773941498.854631:0:14694:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1773941498.854637:0:14694:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d21ec1800. 00010000:00000001:0.0:1773941498.854642:0:14694:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1773941498.854645:0:14694:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d00031400. 00000020:00000010:0.0:1773941498.854649:0:14694:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 128 at ffff988d181fed80. 00000020:00000001:0.0:1773941498.854652:0:14694:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:0.0:1773941498.854654:0:14694:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d00031400 with handle 0x8e86767622e28db to hash 00000020:00000001:0.0:1773941498.854656:0:14694:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:0.0:1773941498.854658:0:14694:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330090853376 : -113743618698240 : ffff988d00031400) 00010000:00000001:0.0:1773941498.854661:0:14694:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330090853376 : -113743618698240 : ffff988d00031400) 00010000:00000001:0.0:1773941498.854665:0:14694:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941498.854668:0:14694:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941498.854673:0:14694:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e28db lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14694 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941498.854683:0:14694:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e28db lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14694 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941498.854695:0:14694:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e28db lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14694 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941498.854702:0:14694:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1773941498.854704:0:14694:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941498.854716:0:14694:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb6600. 00000100:00000001:0.0:1773941498.854719:0:14694:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330139633152 : -113743569918464 : ffff988d02eb6600) 00000100:00000001:0.0:1773941498.854722:0:14694:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941498.854723:0:14694:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367693184 : -113743341858432 : ffff988d10835180) 00000100:00000001:0.0:1773941498.854727:0:14694:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941498.854729:0:14694:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.854732:0:14694:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941498.854733:0:14694:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941498.854739:0:14694:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10835180 x1860112291476864/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941498.854747:0:14694:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941498.854749:0:14694:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1773941498.854754:0:14694:0:(client.c:1829:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff988d10835180 x1860112291476864/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941498.854761:0:14694:0:(client.c:1834:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941498.854763:0:14694:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941498.854765:0:14694:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941498.854768:0:14694:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6600 going to sleep for 0 seconds 00000100:00000001:0.0:1773941498.854770:0:14694:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941498.854772:0:14694:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941498.854774:0:14694:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941498.854775:0:14694:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941498.854778:0:14694:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941498.854796:0:461:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (226) 00000800:00000001:0.0:1773941498.854815:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941498.854820:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941498.854823:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941498.854828:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941498.854831:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941498.854834:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941498.854839:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941498.854844:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : ACK - for me 00000400:00000200:0.0:1773941498.854851:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941498.854855:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941498.854864:0:2393:0:(lib-move.c:4700:lnet_parse_ack()) 192.168.204.14@tcp: ACK from 12345-192.168.204.124@tcp into md 0x81d 00000400:00000200:0.0:1773941498.854875:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941498.854880:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: ACK: OK 00000400:00000200:0.0:1773941498.854886:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac198 00000400:00000010:0.0:1773941498.854889:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac198. 00000400:00000200:0.0:1773941498.854893:0:2393:0:(peer.c:3081:lnet_discovery_event_handler()) Received event: 4 00000400:00000200:0.0:1773941498.854896:0:2393:0:(peer.c:2799:lnet_discovery_event_ack()) peer 192.168.204.114@tcp ev->status 0 00000400:00000200:0.0:1773941498.854900:0:2393:0:(peer.c:2405:lnet_peer_queue_for_discovery()) Queue peer 192.168.204.114@tcp: -114 00000800:00000001:0.0:1773941498.854918:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941498.854923:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941498.854925:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941498.854929:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941498.854933:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:2.0:1773941498.854936:0:2399:0:(peer.c:3988:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1773941498.854941:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:2.0:1773941498.854944:0:2399:0:(peer.c:4100:lnet_peer_discovery()) peer 192.168.204.114@tcp(ffff988d03357600) state 0x100141 00000400:00000200:2.0:1773941498.854950:0:2399:0:(peer.c:3844:lnet_peer_discovered()) peer 192.168.204.114@tcp 00000400:00000200:0.0:1773941498.854951:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:2.0:1773941498.854953:0:2399:0:(peer.c:4118:lnet_peer_discovery()) peer 192.168.204.114@tcp(ffff988d03357600) state 0x100111 rc 0 00000400:00000200:0.0:1773941498.854959:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941498.855002:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941498.855004:0:2399:0:(peer.c:2421:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 192.168.204.114@tcp 00000400:00000200:0.0:1773941498.855011:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 48 into portal 0 MB=0x8000000000000000 00000400:00000200:0.0:1773941498.855020:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 0 from 12345-192.168.204.114@tcp of length 48/48 into md 0x19 [1] + 0 00000400:00000010:0.0:1773941498.855025:0:2393:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d1121e108. 00000400:00000200:0.0:1773941498.855030:0:2393:0:(lib-md.c:42:lnet_md_unlink()) Queueing unlink of md ffff988d04516770 00000800:00000001:0.0:1773941498.855034:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941498.855064:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941498.855067:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941498.855072:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941498.855075:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516770 00000400:00000010:0.0:1773941498.855078:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516770. 00000400:00000200:0.0:1773941498.855082:0:2393:0:(api-ni.c:2341:lnet_push_target_event_handler()) type 2 status 0 unlinked 1 00000400:00000200:0.0:1773941498.855086:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941498.855090:0:2393:0:(peer.c:2570:lnet_peer_push_event()) peer 192.168.204.114@tcp(ffff988d03357600) is MR 00000400:00000010:0.0:1773941498.855095:0:2393:0:(api-ni.c:1821:lnet_ping_buffer_alloc()) alloc '(pbuf)': 57 at ffff988d10872b80. 00000400:00000200:0.0:1773941498.855099:0:2393:0:(peer.c:2644:lnet_peer_push_event()) Received Push 192.168.204.114@tcp 363512030 00000400:00000200:0.0:1773941498.855110:0:2393:0:(peer.c:2405:lnet_peer_queue_for_discovery()) Queue peer 192.168.204.114@tcp: 0 00000400:00000200:0.0:1773941498.855116:0:2393:0:(lib-md.c:96:lnet_cpt_of_md()) Couldn't resolve first page of md 0000000000000000 with offset 0 00000400:00000200:0.0:1773941498.855120:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941498.855131:0:2393:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source Specified: 192.168.204.14@tcp to MR: 192.168.204.124@tcp local destination 00000400:00000200:2.0:1773941498.855134:0:2399:0:(peer.c:3988:lnet_peer_discovery_wait_for_work()) woken: 0 00000400:00000200:0.0:1773941498.855140:0:2393:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [113:113:256:4294967295] 192.168.204.124@tcp LPNI seq info [3:3:8:4294967295] 00000400:00000010:2.0:1773941498.855141:0:2399:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d20379580. 00000400:00000010:2.0:1773941498.855147:0:2399:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516dd0. 00000400:00000200:0.0:1773941498.855149:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:2.0:1773941498.855154:0:2399:0:(api-ni.c:2331:lnet_push_target_post()) posted push target ffff988d0627bc00 00000400:00000200:0.0:1773941498.855162:0:2393:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:192.168.204.14@tcp) -> 192.168.204.124@tcp(192.168.204.124@tcp:192.168.204.124@tcp) 192.168.204.124@tcp : ACK try# 0 00000800:00000200:0.0:1773941498.855170:0:2393:0:(socklnd_cb.c:1001:ksocknal_send()) sending 0 bytes in 1 frags to 12345-192.168.204.124@tcp 00000400:00000200:2.0:1773941498.855172:0:2399:0:(peer.c:4100:lnet_peer_discovery()) peer 192.168.204.114@tcp(ffff988d03357600) state 0x1000d1 00000800:00000010:0.0:1773941498.855177:0:2393:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a20bd00. 00000400:00000200:2.0:1773941498.855179:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:2.0:1773941498.855187:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000800:00000200:0.0:1773941498.855191:0:2393:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000400:00000200:2.0:1773941498.855195:0:2399:0:(peer.c:1899:lnet_peer_add_nid()) peer 192.168.204.114@tcp NID 192.168.204.124@tcp flags 0x11: 0 00000400:00000010:2.0:1773941498.855201:0:2399:0:(peer.c:3265:lnet_peer_merge_data()) alloc '(curnis)': 40 at ffff988d05437080. 00000800:00000200:0.0:1773941498.855204:0:2393:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000400:00000010:2.0:1773941498.855205:0:2399:0:(peer.c:3266:lnet_peer_merge_data()) alloc '(addnis)': 48 at ffff988d05437280. 00000400:00000010:2.0:1773941498.855208:0:2399:0:(peer.c:3267:lnet_peer_merge_data()) alloc '(delnis)': 40 at ffff988d054377c0. 00000800:00000200:0.0:1773941498.855211:0:2393:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a20bd00 type 0, nob 96 niov 1 nkiov 0 00000400:00000200:2.0:1773941498.855223:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:2.0:1773941498.855231:0:2399:0:(peer.c:566:lnet_peer_del_nid()) peer 192.168.204.114@tcp NID 192.168.204.114@tcp flags 0x11: -1 00000400:00020000:2.0:1773941498.855237:0:2399:0:(peer.c:3365:lnet_peer_merge_data()) Error deleting NID 192.168.204.114@tcp from peer 192.168.204.114@tcp: -1 00000400:00000200:2.0:1773941498.855240:0:2399:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000010:2.0:1773941498.855241:0:2399:0:(peer.c:3405:lnet_peer_merge_data()) kfreed 'curnis': 40 at ffff988d05437080. 00000400:00000010:2.0:1773941498.855243:0:2399:0:(peer.c:3406:lnet_peer_merge_data()) kfreed 'addnis': 48 at ffff988d05437280. 00000400:00000010:2.0:1773941498.855245:0:2399:0:(peer.c:3407:lnet_peer_merge_data()) kfreed 'delnis': 40 at ffff988d054377c0. 00000400:00000010:2.0:1773941498.855252:0:2399:0:(api-ni.c:1839:lnet_ping_buffer_free()) kfreed 'pbuf': 57 at ffff988d10872b80. 00000400:00000200:2.0:1773941498.855255:0:2399:0:(peer.c:3410:lnet_peer_merge_data()) peer 192.168.204.114@tcp (ffff988d03357600): 0 00000400:00000200:2.0:1773941498.855258:0:2399:0:(peer.c:3714:lnet_peer_data_present()) peer 192.168.204.114@tcp(ffff988d03357600): 0. state = 0x100151 00000400:00000200:2.0:1773941498.855261:0:2399:0:(peer.c:4118:lnet_peer_discovery()) peer 192.168.204.114@tcp(ffff988d03357600) state 0x100151 rc 1 00000400:00000200:2.0:1773941498.855264:0:2399:0:(peer.c:4100:lnet_peer_discovery()) peer 192.168.204.114@tcp(ffff988d03357600) state 0x100151 00000400:00000200:2.0:1773941498.855266:0:2399:0:(peer.c:3844:lnet_peer_discovered()) peer 192.168.204.114@tcp 00000400:00000200:2.0:1773941498.855269:0:2399:0:(peer.c:4118:lnet_peer_discovery()) peer 192.168.204.114@tcp(ffff988d03357600) state 0x100111 rc 0 00000400:00000200:2.0:1773941498.855274:0:2399:0:(peer.c:2421:lnet_peer_discovery_complete()) Discovery complete. Dequeue peer 192.168.204.114@tcp 00000800:00000200:0.0:1773941498.855276:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941498.855281:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a20bd00. 00000400:00000200:0.0:1773941498.855284:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941498.855290:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: ACK: OK 00000800:00000001:0.0:1773941498.855296:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941498.855300:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1773941498.855314:0:41:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1773941498.855317:0:41:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1773941498.855321:0:41:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 448 at ffff988d065c4600. 00000020:00000001:2.0:1773941498.855357:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941498.855359:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941498.855361:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1773941499.899080:0:41:0:(ldlm_pool.c:458:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1773941499.899085:0:41:0:(ldlm_pool.c:462:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941499.899088:0:2400:0:(client.c:2542:ptlrpc_expired_set()) Process entered 00000100:00000001:2.0:1773941499.899092:0:2400:0:(client.c:2460:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1773941499.899105:0:14694:0:(client.c:2542:ptlrpc_expired_set()) Process entered 00000100:00100000:2.0:1773941499.899107:0:2400:0:(client.c:2478:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1773941498/real 0] req@ffff988d10836a00 x1860112291476736/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 1 dl 1773941499 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1773941499.899109:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000100:00000001:1.0:1773941499.899110:0:14694:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941499.899113:0:14694:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941499.899116:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941499.899120:0:14694:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6600 going to sleep for 0 seconds 00010000:00010000:3.0:1773941499.899124:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1773941499.899126:0:14694:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941499.899128:0:2400:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941499.899131:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941499.899133:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941499.899136:0:2400:0:(client.c:2509:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff988d10836a00 x1860112291476736/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 1 dl 1773941499 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941499.899144:0:14694:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941499.899146:0:2400:0:(client.c:2514:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941499.899148:0:14694:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941499.899151:0:14694:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941499.899154:0:14694:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941499.899155:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941499.899158:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941499.899161:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941499.899162:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941499.899164:0:2400:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941499.899167:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10836a00 x1860112291476736/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 1 dl 1773941499 ref 1 fl Rpc:EXNQU/200/ffffffff rc -110/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941499.899173:0:2400:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1773941499.899175:0:2400:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941499.899176:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941499.899185:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941499.899187:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1773941499.899189:0:2400:0:(import.c:1084:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1773941499.899207:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1773941499.899222:0:2400:0:(import.c:1499:ptlrpc_connect_interpret()) recovery of MGS on 192.168.204.114@tcp failed (-110) 00000100:00000001:2.0:1773941499.899224:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000040:2.0:1773941499.899228:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10836a00 x1860112291476736/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 1 dl 1773941499 ref 1 fl Interpret:EXNQU/200/ffffffff rc -110/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941499.899235:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10836a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291476736:192.168.204.114@tcp:250:mount.lustre.0 00000100:00000001:2.0:1773941499.899239:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941499.899242:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941499.899245:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10836a00 x1860112291476736/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 1 dl 1773941499 ref 1 fl Complete:EXNQU/200/ffffffff rc -110/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1773941499.899249:0:461:0:(pinger.c:229:ptlrpc_pinger_process_import()) c1c1ff21-5f68-4bf6-bdc4-258644b24465->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 0 suppress 0 00000100:00000001:2.0:1773941499.899251:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00080000:1.0:1773941499.899256:0:461:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1773941499.899258:0:461:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941499.899259:0:461:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941499.899262:0:461:0:(import.c:484:import_select_connection()) Process entered 00000020:00000001:2.0:1773941499.899263:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941499.899267:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00080000:1.0:1773941499.899267:0:461:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 221 00000020:00000001:2.0:1773941499.899273:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000400:00000200:1.0:1773941499.899278:0:461:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 02000000:00000010:2.0:1773941499.899279:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d02ce6c00. 02000000:00000001:2.0:1773941499.899285:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000400:00000200:1.0:1773941499.899286:0:461:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 02000000:00000001:2.0:1773941499.899288:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941499.899292:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10836a00. 00000100:00000040:1.0:1773941499.899292:0:461:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941499.899299:0:461:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1773941499.899300:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941499.899304:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941499.899304:0:461:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941499.899309:0:461:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000100:00000001:2.0:1773941499.899310:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1773941499.899316:0:461:0:(genops.c:811:class_conn2export()) Process entered 00000100:00000001:2.0:1773941499.899318:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000020:00000040:1.0:1773941499.899319:0:461:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000100:00000001:2.0:1773941499.899321:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941499.899323:0:461:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1773941499.899325:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:1.0:1773941499.899326:0:461:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000100:00000001:2.0:1773941499.899328:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941499.899331:0:461:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941499.899334:0:461:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941499.899336:0:461:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941499.899338:0:461:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941499.899340:0:461:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941499.899342:0:461:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941499.899344:0:461:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941499.899346:0:461:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941499.899348:0:461:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941499.899350:0:461:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941499.899352:0:461:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941499.899359:0:461:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941499.899362:0:461:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941499.899364:0:461:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941499.899368:0:461:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c2a00. 00000020:00000040:1.0:1773941499.899370:0:461:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941499.899373:0:461:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941499.899375:0:461:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941499.899378:0:461:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941499.899380:0:461:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941499.899383:0:461:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941499.899388:0:461:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d1118b400. 00000100:00000001:1.0:1773941499.899393:0:461:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941499.899399:0:461:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c2a00 x1860112291476992/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941499.899406:0:461:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941499.899410:0:461:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941499.899413:0:461:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fb7523b4] to pc [ptlrpcd_rcv+-1] req@ffff988d203c2a00 x1860112291476992/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941499.899429:0:461:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941499.899432:0:461:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (227) 00000100:00000001:2.0:1773941499.899452:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941499.899455:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941499.899456:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941499.899461:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c2a00 x1860112291476992/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941499.899476:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941499.899478:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941499.899480:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941499.899482:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941499.899484:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941499.899487:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291476992:192.168.204.114@tcp:250:kworker.0 00000100:00000001:2.0:1773941499.899490:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941499.899493:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941499.899495:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941499.899496:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941499.899501:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a6400. 02000000:00000001:2.0:1773941499.899502:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941499.899506:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941499.899510:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000400:00000010:2.0:1773941499.899514:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000200:2.0:1773941499.899519:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291476992, portal 25 00000100:00000001:2.0:1773941499.899521:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941499.899522:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631465472 : -113743078086144 : ffff988d203c2a00) 00000100:00000040:2.0:1773941499.899526:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c2a00 x1860112291476992/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941509 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941499.899531:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941499.899534:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:2.0:1773941499.899536:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516000. 00000100:00000200:2.0:1773941499.899540:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291476992, offset 0 00000400:00000200:2.0:1773941499.899547:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941499.899555:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941499.899561:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941499.899566:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941499.899571:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941499.899575:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941499.899578:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941499.899590:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[0, 3] 00000400:00000200:2.0:1773941499.899593:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:2.0:1773941499.899596:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [114:114:256:4294967295] 192.168.204.114@tcp LPNI seq info [4:4:8:4294967295] 00000400:00000200:2.0:1773941499.899600:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941499.899609:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:2.0:1773941499.899613:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:2.0:1773941499.899617:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0dbb00. 00000800:00000200:2.0:1773941499.899623:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:2.0:1773941499.899629:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:2.0:1773941499.899633:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0dbb00 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:2.0:1773941499.899645:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941499.899648:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941499.899650:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294891495 00000100:00000001:2.0:1773941499.899655:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294890375 00000100:00000001:2.0:1773941499.899660:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294891495 00000100:00000001:2.0:1773941499.899664:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294890375 00000100:00000001:2.0:1773941499.899669:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941499.899671:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941499.899673:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941499.899675:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941499.899678:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941499.899679:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941499.899680:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941499.899681:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941499.899682:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941499.899684:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941499.899766:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941499.899770:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0dbb00. 00000400:00000200:0.0:1773941499.899774:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941499.899779:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941499.899783:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516000 00000400:00000010:0.0:1773941499.899785:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516000. 00000100:00000001:0.0:1773941499.899788:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941499.899792:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c2a00 x1860112291476992/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941509 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941499.899799:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941499.899801:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c2a00 x1860112291476992/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941509 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941499.899812:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941499.899813:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941499.902095:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941499.902106:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941499.902108:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941499.902110:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941499.902114:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941499.902117:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941499.902122:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941499.902124:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941499.902128:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941499.902135:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324603600 00000400:00000200:0.0:1773941499.902138:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941499.902142:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xad9 [1] + 0 00000800:00000001:0.0:1773941499.902147:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941499.902154:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941499.902155:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941499.902158:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941499.902161:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941499.902165:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c2a00 x1860112291476992/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941509 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941499.902173:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c2a00 x1860112291476992/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941509 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941499.902186:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941499.902202:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941499.902205:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941499.902223:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941499.902226:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941499.902229:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941499.902231:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941499.902232:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941499.902235:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941499.902241:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941499.902243:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941499.902248:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c2a00 x1860112291476992/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941509 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941499.902255:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c2a00 x1860112291476992/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941509 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941499.902260:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941499.902261:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941499.902262:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941499.902264:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941499.902266:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941499.902267:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941499.902269:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941499.902274:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=1773941499 v=5 (0 0 0 0) 00000100:00001000:2.0:1773941499.902276:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941499 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941499.902278:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000100:00000001:2.0:1773941499.902281:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941499.902282:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941499.902284:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941499.902286:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941499.902288:0:2400:0:(client.c:3065:ptlrpc_free_committed()) Process entered 00000100:00100000:2.0:1773941499.902289:0:2400:0:(client.c:3081:ptlrpc_free_committed()) MGC192.168.204.114@tcp: committing for last_committed 0 gen 1 00000100:00000040:2.0:1773941499.902291:0:2400:0:(client.c:3191:ptlrpc_free_committed()) MGC192.168.204.114@tcp: normal: skip=0 replay=0/0 committed=0/0 00000100:00000001:2.0:1773941499.902293:0:2400:0:(client.c:3192:ptlrpc_free_committed()) Process leaving 00000100:00000001:2.0:1773941499.902294:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941499.902298:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c2a00 x1860112291476992/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941509 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941499.902302:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941499.902303:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941499.902304:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941499.902305:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941499.902309:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941499.902311:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941499.902312:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941499.902316:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941499.902318:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941499.902320:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941499.902321:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:2.0:1773941499.902324:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941499 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941499.902326:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000100:00080000:2.0:1773941499.902330:0:2400:0:(import.c:951:ptlrpc_connect_set_flags()) MGC192.168.204.114@tcp: Resetting ns_connect_flags to server flags: 0xa000011001002060 00000020:00000040:2.0:1773941499.902332:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:2.0:1773941499.902334:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941499.902337:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941499.902338:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941499.902339:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941499.902342:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to FULL 00000100:00000001:2.0:1773941499.902344:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941499.902344:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:2.0:1773941499.902346:0:2400:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:2.0:1773941499.902357:0:2400:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:2.0:1773941499.902358:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941499.902359:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941499.902360:0:2400:0:(import.c:1232:ptlrpc_connect_interpret()) Process leaving via finish (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941499.902363:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941499.902366:0:2400:0:(recover.c:189:ptlrpc_wake_delayed()) @@@ waking (set 0000000016bffbe1): req@ffff988d10835180 x1860112291476864/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941499.902376:0:2400:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941499.902379:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941499.902382:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c2a00 x1860112291476992/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941509 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941499.902388:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291476992:192.168.204.114@tcp:250:kworker.0 00000100:00000001:2.0:1773941499.902393:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941499.902395:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941499.902400:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c2a00 x1860112291476992/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941509 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941499.902413:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941499.902416:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941499.902419:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a6400. 02000000:00000001:2.0:1773941499.902423:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941499.902424:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941499.902426:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941499.902428:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941499.902430:0:14694:0:(client.c:1974:ptlrpc_check_set()) Process entered 02000000:00000010:2.0:1773941499.902431:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d1118b400. 02000000:00000001:2.0:1773941499.902435:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941499.902437:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941499.902440:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c2a00. 00000100:00000001:1.0:1773941499.902440:0:14694:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941499.902443:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941499.902445:0:14694:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941499.902446:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941499.902449:0:14694:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941499.902450:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941499.902458:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:1.0:1773941499.902458:0:14694:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1773941499.902462:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941499.902464:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1773941499.902464:0:14694:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941499.902467:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941499.902471:0:14694:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941499.902475:0:14694:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941499.902479:0:14694:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941499.902481:0:14694:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941499.902484:0:14694:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941499.902489:0:14694:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d1118a400. 02000000:00000001:1.0:1773941499.902492:0:14694:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941499.902497:0:14694:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941499.902501:0:14694:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c5d8. 00000400:00000010:1.0:1773941499.902513:0:14694:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000100:00000200:1.0:1773941499.902517:0:14694:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291476864, portal 25 00000100:00000001:1.0:1773941499.902519:0:14694:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941499.902521:0:14694:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367693184 : -113743341858432 : ffff988d10835180) 00000100:00000040:1.0:1773941499.902528:0:14694:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10835180 x1860112291476864/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941515 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941499.902536:0:14694:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941499.902539:0:14694:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941499.902541:0:14694:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113dea18. 00000100:00000200:1.0:1773941499.902544:0:14694:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291476864, offset 0 00000400:00000200:1.0:1773941499.902549:0:14694:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941499.902555:0:14694:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941499.902559:0:14694:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941499.902563:0:14694:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941499.902566:0:14694:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941499.902569:0:14694:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941499.902572:0:14694:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941499.902575:0:14694:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[4, 3] 00000400:00000200:1.0:1773941499.902579:0:14694:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:1.0:1773941499.902583:0:14694:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [115:115:256:4294967295] 192.168.204.124@tcp LPNI seq info [5:5:8:4294967295] 00000400:00000200:1.0:1773941499.902588:0:14694:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941499.902595:0:14694:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:1.0:1773941499.902601:0:14694:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:1.0:1773941499.902605:0:14694:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfd200. 00000800:00000200:1.0:1773941499.902610:0:14694:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:1.0:1773941499.902616:0:14694:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:1.0:1773941499.902620:0:14694:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfd200 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941499.902633:0:14694:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941499.902637:0:14694:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941499.902640:0:14694:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294891498 00000100:00000001:1.0:1773941499.902647:0:14694:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294891495 00000100:00000001:1.0:1773941499.902659:0:14694:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294891498 00000100:00000001:1.0:1773941499.902664:0:14694:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294891495 00000100:00000001:1.0:1773941499.902669:0:14694:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941499.902673:0:14694:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941499.902676:0:14694:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941499.902678:0:14694:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941499.902680:0:14694:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6600 going to sleep for 11 seconds 00000100:00000001:1.0:1773941499.902684:0:14694:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941499.902686:0:14694:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941499.902686:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941499.902690:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfd200. 00000400:00000200:0.0:1773941499.902695:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941499.902699:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941499.902701:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113dea18 00000400:00000010:0.0:1773941499.902708:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113dea18. 00000100:00000001:0.0:1773941499.902711:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941499.902715:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10835180 x1860112291476864/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941515 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941499.902721:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941499.902723:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d10835180 x1860112291476864/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941515 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941499.902728:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941499.902729:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941499.903427:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941499.903433:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941499.903434:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941499.903436:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941499.903439:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941499.903443:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941499.903446:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941499.903449:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941499.903451:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941499.903453:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324603580 00000400:00000200:0.0:1773941499.903456:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941499.903459:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xae1 [1] + 224 00000800:00000001:0.0:1773941499.903462:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941499.903466:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941499.903468:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941499.903470:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941499.903473:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941499.903476:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10835180 x1860112291476864/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941515 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941499.903483:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d10835180 x1860112291476864/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941515 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941499.903498:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941499.903510:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941499.903513:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941499.903539:0:14694:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941499.903544:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941499.903546:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941499.903548:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:1.0:1773941499.903552:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:1.0:1773941499.903554:0:14694:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:1.0:1773941499.903557:0:14694:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941499.903563:0:14694:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10835180 x1860112291476864/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941515 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941499.903575:0:14694:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10835180 x1860112291476864/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941515 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941499.903582:0:14694:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941499.903584:0:14694:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941499.903586:0:14694:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941499.903588:0:14694:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941499.903590:0:14694:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941499.903592:0:14694:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941499.903594:0:14694:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941499.903598:0:14694:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=0 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941499.903601:0:14694:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=0 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941499.903604:0:14694:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941499.903607:0:14694:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:1.0:1773941499.906521:0:14694:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941499.906525:0:14694:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941499.906533:0:14694:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941499.906536:0:14694:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941499.906539:0:14694:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941499.909326:0:14694:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941499.909328:0:14694:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941499.909330:0:14694:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941499.909331:0:14694:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941499.909336:0:14694:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d10835180 x1860112291476864/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941515 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941499.909343:0:14694:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941499.909344:0:14694:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941499.909346:0:14694:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941499.909347:0:14694:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941499.909349:0:14694:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941499.909350:0:14694:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941499.909352:0:14694:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941499.909353:0:14694:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941499.909354:0:14694:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941499.909355:0:14694:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941499.909356:0:14694:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941499.909358:0:14694:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941499.909359:0:14694:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941499.909361:0:14694:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941499.909362:0:14694:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 222 00000400:00000200:1.0:1773941499.909365:0:14694:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941499.909368:0:14694:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941499.909370:0:14694:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941499.909372:0:14694:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941499.909373:0:14694:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941499.909374:0:14694:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941499.909376:0:14694:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941499.909377:0:14694:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941499.909379:0:14694:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941499.909386:0:14694:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941499.909388:0:14694:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941499.909389:0:14694:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941499.909392:0:14694:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941499.909393:0:14694:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941499.909395:0:14694:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941499.909398:0:14694:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941499.909400:0:14694:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941499.909402:0:14694:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941499.909404:0:14694:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941499.909406:0:14694:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941499.909407:0:14694:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941499.909409:0:14694:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941499.909410:0:14694:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941499.909411:0:14694:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941499.909414:0:14694:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c3480. 00000020:00000040:1.0:1773941499.909416:0:14694:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941499.909419:0:14694:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941499.909420:0:14694:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941499.909422:0:14694:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941499.909423:0:14694:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941499.909425:0:14694:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941499.909429:0:14694:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d1118a800. 00000100:00000001:1.0:1773941499.909432:0:14694:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941499.909435:0:14694:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c3480 x1860112291477120/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941499.909441:0:14694:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941499.909443:0:14694:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941499.909445:0:14694:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000099ddc7e5] to pc [ptlrpcd_rcv+-1] req@ffff988d203c3480 x1860112291477120/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941499.909461:0:14694:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941499.909463:0:14694:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941499.909468:0:14694:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941499.909470:0:14694:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941499.909472:0:14694:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941499.909473:0:14694:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941499.909474:0:14694:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6600 going to sleep for 1 seconds 00000100:00000001:1.0:1773941499.909476:0:14694:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941499.909478:0:14694:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941499.909481:0:14694:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10835180 x1860112291476864/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941515 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941499.909486:0:14694:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941499.909488:0:14694:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941499.909490:0:14694:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941499.909491:0:14694:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941499.909494:0:14694:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10835180 x1860112291476864/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941515 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941499.909500:0:14694:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10835180 pname:cluuid:pid:xid:nid:opc:job mount.lustre:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14694:1860112291476864:192.168.204.114@tcp:101:mount.lustre.0 00000100:00000001:3.0:1773941499.909504:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941499.909505:0:14694:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941499.909510:0:14694:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941499.909511:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941499.909512:0:14694:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:3.0:1773941499.909515:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941499.909516:0:14694:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941499.909524:0:14694:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10835180 x1860112291476864/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941515 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941499.909529:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c3480 x1860112291477120/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941499.909530:0:14694:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941499.909534:0:14694:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb6600. 00000100:00000001:1.0:1773941499.909539:0:14694:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941499.909542:0:14694:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:1.0:1773941499.909548:0:14694:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000001:3.0:1773941499.909550:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:1.0:1773941499.909551:0:14694:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000001:3.0:1773941499.909553:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941499.909554:0:14694:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941499.909557:0:14694:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00031400 refcount=4 02000000:00000001:3.0:1773941499.909560:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:1.0:1773941499.909560:0:14694:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090853376 : -113743618698240 : ffff988d00031400) 02000000:00000001:3.0:1773941499.909564:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941499.909568:0:14694:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090853376 : -113743618698240 : ffff988d00031400) 02000000:00000001:3.0:1773941499.909570:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941499.909574:0:14694:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e28db lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14694 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1773941499.909577:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291477120:192.168.204.114@tcp:250:mount.lustre.0 00000100:00000001:3.0:1773941499.909587:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00000001:1.0:1773941499.909589:0:14694:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 02000000:00000001:3.0:1773941499.909592:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00010000:1.0:1773941499.909594:0:14694:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e28db lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14694 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941499.909596:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941499.909602:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:1.0:1773941499.909604:0:14694:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941499.909607:0:14694:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e28db lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14694 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000010:3.0:1773941499.909609:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03522c00. 00010000:00000001:1.0:1773941499.909613:0:14694:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 02000000:00000001:3.0:1773941499.909615:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941499.909616:0:14694:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941499.909619:0:14694:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e28db lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14694 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941499.909625:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941499.909628:0:14694:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000010:3.0:1773941499.909632:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00010000:00000001:1.0:1773941499.909632:0:14694:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941499.909637:0:14694:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000010:3.0:1773941499.909641:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00010000:00010000:1.0:1773941499.909641:0:14694:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e28db lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14694 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:3.0:1773941499.909650:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291477120, portal 25 00010000:00010000:1.0:1773941499.909650:0:14694:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d00031400 already unused, calling callback (ffffffffc12b4c20) 10000000:00000001:1.0:1773941499.909655:0:14694:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000001:3.0:1773941499.909656:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 10000000:00010000:1.0:1773941499.909658:0:14694:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e28db lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14694 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941499.909660:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631468160 : -113743078083456 : ffff988d203c3480) 00010000:00000001:1.0:1773941499.909669:0:14694:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941499.909671:0:14694:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000040:3.0:1773941499.909672:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c3480 x1860112291477120/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941509 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1773941499.909672:0:14694:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941499.909675:0:14694:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00031400 refcount=5 00000020:00000001:1.0:1773941499.909682:0:14694:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090853376 : -113743618698240 : ffff988d00031400) 00010000:00000001:1.0:1773941499.909686:0:14694:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090853376 : -113743618698240 : ffff988d00031400) 00000100:00000001:3.0:1773941499.909691:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:1.0:1773941499.909691:0:14694:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941499.909695:0:14694:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e28db lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14694 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:3.0:1773941499.909699:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 10000000:00000001:1.0:1773941499.909705:0:14694:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000010:3.0:1773941499.909706:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac770. 10000000:00010000:1.0:1773941499.909708:0:14694:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e28db lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14694 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:3.0:1773941499.909713:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291477120, offset 0 10000000:01000000:1.0:1773941499.909715:0:14694:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941499.909720:0:14694:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941499.909721:0:14694:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941499.909725:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00010000:1.0:1773941499.909726:0:14694:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941499.909729:0:14694:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941499.909733:0:14694:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000400:00000200:3.0:1773941499.909735:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941499.909736:0:14694:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941499.909740:0:14694:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941499.909742:0:14694:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941499.909746:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000020:00000040:1.0:1773941499.909747:0:14694:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d00031400 with handle 0x8e86767622e28db from hash 00010000:00000001:1.0:1773941499.909753:0:14694:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000400:00000200:3.0:1773941499.909755:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:1.0:1773941499.909757:0:14694:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941499.909759:0:14694:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941499.909760:0:14694:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941499.909762:0:14694:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941499.909764:0:14694:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000400:00000200:3.0:1773941499.909770:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00000001:1.0:1773941499.909771:0:14694:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941499.909776:0:14694:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:3.0:1773941499.909779:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941499.909780:0:14694:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941499.909786:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 10000000:00000001:1.0:1773941499.909786:0:14694:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941499.909794:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[4, 5] 00010000:00010000:1.0:1773941499.909795:0:14694:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e28db lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14694 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941499.909803:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00010000:00000001:1.0:1773941499.909810:0:14694:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:3.0:1773941499.909811:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [116:116:256:4294967295] 192.168.204.114@tcp LPNI seq info [6:6:8:4294967295] 00000400:00000200:3.0:1773941499.909817:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941499.909822:0:14694:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941499.909825:0:14694:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941499.909828:0:14694:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941499.909831:0:14694:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941499.909834:0:14694:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941499.909836:0:14694:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:3.0:1773941499.909837:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00010000:00010000:1.0:1773941499.909839:0:14694:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e28db lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14694 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941499.909849:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000020:00000010:1.0:1773941499.909851:0:14694:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 128 at ffff988d0716bc00. 00010000:00000040:1.0:1773941499.909855:0:14694:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d21ec1800 count: 0 00000800:00000010:3.0:1773941499.909857:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74400. 00000800:00000200:3.0:1773941499.909865:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00010000:00000001:1.0:1773941499.909866:0:14694:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941499.909869:0:14694:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941499.909873:0:14694:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000800:00000200:3.0:1773941499.909874:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000100:00000040:1.0:1773941499.909880:0:14694:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10835180 x1860112291476864/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941515 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1773941499.909881:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74400 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:1.0:1773941499.909892:0:14694:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941499.909894:0:14694:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941499.909895:0:14694:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d1118a400. 02000000:00000001:1.0:1773941499.909897:0:14694:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941499.909898:0:14694:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941499.909899:0:14694:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941499.909901:0:14694:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941499.909902:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941499.909903:0:14694:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065c5a00. 02000000:00000001:1.0:1773941499.909910:0:14694:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941499.909912:0:14694:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000400:00000200:3.0:1773941499.909913:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000010:1.0:1773941499.909914:0:14694:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10835180. 00000100:00000001:3.0:1773941499.909920:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294891505 00000100:00000001:1.0:1773941499.909921:0:14694:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941499.909923:0:14694:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941499.909927:0:14694:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941499.909929:0:14694:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941499.909931:0:14694:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000100:00000001:3.0:1773941499.909932:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294891498 10000000:00000010:1.0:1773941499.909934:0:14694:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d04d67c80. 00000100:00000001:3.0:1773941499.909939:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294891505 00000020:00000010:1.0:1773941499.909941:0:14694:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d0a130000. 00000100:00000001:3.0:1773941499.909948:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294891498 00000020:00000010:1.0:1773941499.909949:0:14694:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d0642c200. 00000100:00000001:3.0:1773941499.909957:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000040:00000010:1.0:1773941499.909957:0:14694:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d02cbc200. 00000800:00000200:0.0:1773941499.909986:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941499.909990:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74400. 00000100:00000001:3.0:1773941499.909994:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941499.909996:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1773941499.910000:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941499.910005:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:3.0:1773941499.910006:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941499.910009:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac770 00000400:00000010:0.0:1773941499.910013:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac770. 00000100:00000001:3.0:1773941499.910014:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941499.910019:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1773941499.910019:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000020:00000010:1.0:1773941499.910024:0:14694:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d18b18000. 00000100:00000001:3.0:1773941499.910025:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000200:0.0:1773941499.910026:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c3480 x1860112291477120/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941509 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941499.910029:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000020:00000010:1.0:1773941499.910030:0:14694:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d181fd400. 00000100:00000001:3.0:1773941499.910034:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 80000000:00000010:1.0:1773941499.910038:0:14694:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d054f3300. 00040000:00000010:1.0:1773941499.910045:0:14694:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d181fc000. 00000100:00000001:0.0:1773941499.910045:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941499.910050:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c3480 x1860112291477120/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941509 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 20000000:00000010:1.0:1773941499.910051:0:14694:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d181fc600. 10000000:00000040:1.0:1773941499.910057:0:14694:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 00000100:00000001:0.0:1773941499.910060:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941499.910061:0:14694:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:0.0:1773941499.910063:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:1.0:1773941499.910067:0:14694:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d0642c200. 00000040:00000010:1.0:1773941499.910072:0:14694:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d02cbc200. 00000020:00000010:1.0:1773941499.910075:0:14694:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941499.910079:0:14694:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d181fd400. 80000000:00000010:1.0:1773941499.910082:0:14694:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d054f3300. 00040000:00000010:1.0:1773941499.910085:0:14694:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d181fc000. 20000000:00000010:1.0:1773941499.910088:0:14694:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d181fc600. 00000100:00000001:3.0:1773941499.910091:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941499.910091:0:14694:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0a130000. 10000000:00000010:1.0:1773941499.910097:0:14694:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d04d67c80. 10000000:01000000:1.0:1773941499.910101:0:14694:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941499.910103:0:14694:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941499.910105:0:14694:0:(mgc_request.c:233:do_config_log_add()) MGC192.168.204.114@tcp: failed processing log, type 1: rc = -107 10000000:00000001:1.0:1773941499.913066:0:14694:0:(mgc_request.c:236:do_config_log_add()) Process leaving (rc=18446630330197362688 : -113743512188928 : ffff988d065c4800) 10000000:00000001:1.0:1773941499.913070:0:14694:0:(mgc_request.c:159:config_log_find()) Process entered 10000000:00000001:1.0:1773941499.913071:0:14694:0:(mgc_request.c:177:config_log_find()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941499.913072:0:14694:0:(mgc_request.c:191:do_config_log_add()) Process entered 10000000:01000000:1.0:1773941499.913073:0:14694:0:(mgc_request.c:194:do_config_log_add()) do adding config log params-ffff988d111b1800 10000000:00000010:1.0:1773941499.913076:0:14694:0:(mgc_request.c:197:do_config_log_add()) kmalloced '(cld)': 279 at ffff988d181fc600. 10000000:01000000:1.0:1773941499.913079:0:14694:0:(mgc_request.c:68:mgc_name2resid()) log params to resid 0x736d61726170/0x3 (params) 00000020:00000040:1.0:1773941499.913081:0:14694:0:(genops.c:887:class_export_get()) GET export ffff988d111b5800 refcount=4 10000000:00000001:1.0:1773941499.913083:0:14694:0:(mgc_request.c:236:do_config_log_add()) Process leaving (rc=18446630330495387136 : -113743214164480 : ffff988d181fc600) 10000000:00000001:1.0:1773941499.913085:0:14694:0:(mgc_request.c:191:do_config_log_add()) Process entered 10000000:01000000:1.0:1773941499.913086:0:14694:0:(mgc_request.c:194:do_config_log_add()) do adding config log lustre-client-ffff988d111b1800 10000000:00000010:1.0:1773941499.913087:0:14694:0:(mgc_request.c:197:do_config_log_add()) kmalloced '(cld)': 286 at ffff988d181fc000. 10000000:01000000:1.0:1773941499.913089:0:14694:0:(mgc_request.c:68:mgc_name2resid()) log lustre-client to resid 0x65727473756c/0x0 (lustre) 00000020:00000040:1.0:1773941499.913095:0:14694:0:(genops.c:887:class_export_get()) GET export ffff988d111b5800 refcount=5 10000000:00000001:1.0:1773941499.913096:0:14694:0:(mgc_request.c:236:do_config_log_add()) Process leaving (rc=18446630330495385600 : -113743214166016 : ffff988d181fc000) 10000000:00000001:1.0:1773941499.913098:0:14694:0:(mgc_request.c:191:do_config_log_add()) Process entered 10000000:01000000:1.0:1773941499.913099:0:14694:0:(mgc_request.c:194:do_config_log_add()) do adding config log lustre-cliir-ffff988d111b1800 10000000:00000010:1.0:1773941499.913101:0:14694:0:(mgc_request.c:197:do_config_log_add()) kmalloced '(cld)': 285 at ffff988d181fd400. 10000000:01000000:1.0:1773941499.913102:0:14694:0:(mgc_request.c:68:mgc_name2resid()) log lustre-cliir to resid 0x65727473756c/0x2 (lustre) 00000020:00000040:1.0:1773941499.913104:0:14694:0:(genops.c:887:class_export_get()) GET export ffff988d111b5800 refcount=6 10000000:00000001:1.0:1773941499.913105:0:14694:0:(mgc_request.c:236:do_config_log_add()) Process leaving (rc=18446630330495390720 : -113743214160896 : ffff988d181fd400) 10000000:00000001:1.0:1773941499.913106:0:14694:0:(mgc_request.c:441:config_log_add()) Process leaving (rc=18446630330495385600 : -113743214166016 : ffff988d181fc000) 00000800:00000001:0.0:1773941499.913692:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941499.913698:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941499.913699:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941499.913701:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941499.913705:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941499.913708:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941499.913712:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941499.913714:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941499.913717:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941499.913719:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324603680 00000400:00000200:0.0:1773941499.913722:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941499.913725:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xaf1 [1] + 0 00000800:00000001:0.0:1773941499.913728:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941499.913733:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941499.913734:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941499.913737:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941499.913739:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941499.913748:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c3480 x1860112291477120/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941509 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941499.913756:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c3480 x1860112291477120/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941509 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941499.913768:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941499.913772:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941499.913775:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941499.913808:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941499.913811:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941499.913815:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941499.913823:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941499.913826:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941499.913831:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941499.913833:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941499.913836:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941499.913843:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c3480 x1860112291477120/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941509 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941499.913856:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c3480 x1860112291477120/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941509 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941499.913865:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941499.913867:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941499.913870:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941499.913872:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941499.913874:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941499.913876:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941499.913878:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941499.913882:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=0 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941499.913886:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=0 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941499.913889:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941499.913890:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941499.913892:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941499.913894:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941499.913897:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941499.913902:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c3480 x1860112291477120/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941509 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941499.913909:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941499.913912:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941499.913913:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941499.913915:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941499.913919:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941499.913921:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941499.913923:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941499.913925:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941499.913927:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941499.913929:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941499.913938:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941499.913941:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941499 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941499.913945:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941499.913949:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941499.913951:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941499.913953:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941499.913955:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941499.913957:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941499.913987:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941499.913990:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941499.913993:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941499.913997:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941499.914001:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941499.914004:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941499.914173:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941499.914177:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941499.914184:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c3480 x1860112291477120/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941509 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941499.914186:0:14713:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00080000:1.0:1773941499.914203:0:14713:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:1.0:1773941499.914206:0:14713:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00100000:3.0:1773941499.914212:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291477120:192.168.204.114@tcp:250:mount.lustre.0 00000100:00000001:3.0:1773941499.914225:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941499.914228:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1773941499.914230:0:14713:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941499.914233:0:14713:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000040:3.0:1773941499.914235:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c3480 x1860112291477120/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941509 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000001:00000001:1.0:1773941499.914236:0:14713:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941499.914239:0:14713:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941499.914243:0:14713:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941499.914246:0:14713:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941499.914248:0:14713:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941499.914250:0:14713:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941499.914254:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 10000000:00000001:1.0:1773941499.914254:0:14713:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941499.914259:0:14713:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000001:3.0:1773941499.914260:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00080000:1.0:1773941499.914262:0:14713:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 02000000:00000010:3.0:1773941499.914265:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03522c00. 00000100:00000001:1.0:1773941499.914266:0:14713:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941499.914269:0:14713:0:(recover.c:147:ptlrpc_resend()) Process entered 02000000:00000001:3.0:1773941499.914272:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:1.0:1773941499.914272:0:14713:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941499.914277:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000100:00080000:1.0:1773941499.914277:0:14713:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000020:00000040:3.0:1773941499.914282:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941499.914283:0:14713:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941499.914285:0:14713:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000020:00000001:3.0:1773941499.914290:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 10000000:00000040:1.0:1773941499.914290:0:14713:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 02000000:00000010:3.0:1773941499.914295:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d1118a800. 00000100:00080000:1.0:1773941499.914305:0:14713:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 02000000:00000001:3.0:1773941499.914306:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 10000000:00000001:1.0:1773941499.914308:0:14713:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941499.914311:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1773941499.914311:0:14713:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000010:3.0:1773941499.914316:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c3480. 00000100:02000000:1.0:1773941499.914317:0:14713:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:3.0:1773941499.914323:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941499.914327:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941499.914331:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941499.914336:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941499.914338:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941499.914341:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941499.914343:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941499.917092:0:14713:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941499.917100:0:14713:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941499.917101:0:14713:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941499.917103:0:14713:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941499.917104:0:14713:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941499.918990:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d21ec1800. 00010000:00000010:1.1:1773941499.918997:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d00031400. 10000000:00000001:2.0:1773941500.731046:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:2.0:1773941500.731052:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941500.731056:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941500.731058:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941500.731059:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941500.731060:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:2.0:1773941500.731062:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:2.0:1773941500.731064:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:2.0:1773941500.731066:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:2.0:1773941500.731068:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:2.0:1773941500.731070:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:2.0:1773941500.731080:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06507480. 00000020:00000040:2.0:1773941500.731084:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941500.731088:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941500.731091:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941500.731095:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941500.731102:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d0381f200. 00000100:00000001:2.0:1773941500.731107:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941500.731125:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:2.0:1773941500.731128:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:2.0:1773941500.731134:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d071ee100. 00010000:00000001:2.0:1773941500.731139:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:2.0:1773941500.731142:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d089dfa00. 00000020:00000010:2.0:1773941500.731145:0:14708:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 128 at ffff988d107f4c00. 00000020:00000001:2.0:1773941500.731149:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:2.0:1773941500.731151:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d089dfa00 with handle 0x8e86767622e28e2 to hash 00000020:00000001:2.0:1773941500.731152:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:2.0:1773941500.731153:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941500.731155:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941500.731159:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941500.731162:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941500.731165:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28e2 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941500.731174:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28e2 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941500.731213:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28e2 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941500.731218:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:2.0:1773941500.731219:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:2.0:1773941500.731224:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d09350540. 00000100:00000001:2.0:1773941500.731226:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330245121344 : -113743464430272 : ffff988d09350540) 00000100:00000001:2.0:1773941500.731228:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941500.731229:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196587648 : -113743512963968 : ffff988d06507480) 00000100:00000001:2.0:1773941500.731233:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941500.731237:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941500.731238:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:2.0:1773941500.731240:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941500.731249:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06507480 x1860112291477248/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941500.731265:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941500.731266:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941500.731269:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941500.731272:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941500.731273:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941500.731276:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06507480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291477248:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941500.731279:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941500.731284:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941500.731286:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941500.731287:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941500.731291:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a4800. 02000000:00000001:2.0:1773941500.731292:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941500.731306:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941500.731309:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000400:00000010:2.0:1773941500.731315:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000200:2.0:1773941500.731321:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291477248, portal 25 00000100:00000001:2.0:1773941500.731322:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941500.731323:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196587648 : -113743512963968 : ffff988d06507480) 00000100:00000040:2.0:1773941500.731327:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06507480 x1860112291477248/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941516 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941500.731333:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941500.731335:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:2.0:1773941500.731336:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516aa0. 00000100:00000200:2.0:1773941500.731339:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291477248, offset 0 00000400:00000200:2.0:1773941500.731347:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941500.731354:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941500.731367:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941500.731373:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941500.731377:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941500.731381:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941500.731384:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941500.731393:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[6, 5] 00000400:00000200:2.0:1773941500.731398:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:2.0:1773941500.731402:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [117:117:256:4294967295] 192.168.204.124@tcp LPNI seq info [7:7:8:4294967295] 00000400:00000200:2.0:1773941500.731408:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:2.0:1773941500.731419:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:2.0:1773941500.731427:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:2.0:1773941500.731432:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0dbd00. 00000800:00000200:2.0:1773941500.731439:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:2.0:1773941500.731447:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:2.0:1773941500.731453:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0dbd00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:2.0:1773941500.731474:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941500.731483:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941500.731486:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294892327 00000100:00000001:2.0:1773941500.731494:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294891505 00000100:00000001:2.0:1773941500.731499:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294892327 00000100:00000001:2.0:1773941500.731502:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294891505 00000100:00000001:2.0:1773941500.731512:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941500.731515:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941500.731516:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941500.731518:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:2.0:1773941500.731520:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d09350540 going to sleep for 11 seconds 00000100:00000001:2.0:1773941500.731522:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941500.731525:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941500.731593:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941500.731598:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0dbd00. 00000400:00000200:0.0:1773941500.731602:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941500.731607:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941500.731610:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516aa0 00000400:00000010:0.0:1773941500.731612:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516aa0. 00000100:00000001:0.0:1773941500.731615:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941500.731619:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06507480 x1860112291477248/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941516 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941500.731627:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941500.731629:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d06507480 x1860112291477248/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941516 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941500.731635:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941500.731636:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941500.732681:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941500.732702:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941500.732706:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941500.732710:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941500.732719:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941500.732728:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941500.732738:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941500.732744:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941500.732750:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941500.732755:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324603700 00000400:00000200:0.0:1773941500.732760:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941500.732777:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xaf9 [1] + 224 00000800:00000001:0.0:1773941500.732784:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941500.732793:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941500.732795:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941500.732798:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941500.732804:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941500.732812:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06507480 x1860112291477248/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941516 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941500.732838:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d06507480 x1860112291477248/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941516 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941500.732860:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941500.732870:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941500.732873:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941500.732895:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941500.732901:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941500.732903:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941500.732905:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941500.732908:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941500.732909:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941500.732911:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941500.732916:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06507480 x1860112291477248/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941516 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941500.732935:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06507480 x1860112291477248/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941516 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941500.732940:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941500.732942:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941500.732944:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941500.732946:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941500.732948:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941500.732949:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941500.732951:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941500.732955:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=1 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941500.732958:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941500.732976:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:2.0:1773941500.732980:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:2.0:1773941500.736575:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941500.736583:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:2.0:1773941500.736588:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1773941500.736594:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1773941500.736599:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1773941500.740729:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941500.740735:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941500.740737:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941500.740740:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:2.0:1773941500.740749:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d06507480 x1860112291477248/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941516 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941500.740761:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941500.740763:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941500.740765:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941500.740769:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941500.740772:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:2.0:1773941500.740775:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941500.740777:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941500.740794:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941500.740795:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941500.740797:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941500.740799:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941500.740801:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1773941500.740804:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1773941500.740809:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941500.740813:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 222 00000400:00000200:2.0:1773941500.740822:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941500.740829:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:2.0:1773941500.740834:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941500.740837:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941500.740840:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941500.740843:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:2.0:1773941500.740846:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941500.740848:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941500.740851:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941500.740853:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941500.740855:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941500.740858:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:2.0:1773941500.740861:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941500.740863:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941500.740865:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941500.740867:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:2.0:1773941500.740870:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:2.0:1773941500.740874:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:2.0:1773941500.740877:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941500.740880:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941500.740884:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941500.740887:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941500.740890:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941500.740892:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941500.740904:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06505180. 00000020:00000040:2.0:1773941500.740909:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:2.0:1773941500.740913:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941500.740915:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941500.740918:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941500.740920:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941500.740925:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941500.740933:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d035a5c00. 00000100:00000001:2.0:1773941500.740939:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941500.740946:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d06505180 x1860112291477376/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941500.740956:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941500.740986:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941500.740993:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bc0cdaa] to pc [ptlrpcd_rcv+-1] req@ffff988d06505180 x1860112291477376/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941500.741014:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941500.741017:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:2.0:1773941500.741019:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941500.741024:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941500.741027:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941500.741029:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941500.741032:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d09350540 going to sleep for 1 seconds 00000100:00000001:2.0:1773941500.741035:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941500.741038:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941500.741043:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06507480 x1860112291477248/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941516 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941500.741053:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941500.741056:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941500.741060:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941500.741062:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941500.741067:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06507480 x1860112291477248/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941516 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941500.741082:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941500.741091:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1773941500.741095:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06507480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291477248:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941500.741097:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1773941500.741106:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941500.741112:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:3.0:1773941500.741113:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06505180 x1860112291477376/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941500.741116:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:2.0:1773941500.741120:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941500.741130:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06507480 x1860112291477248/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941516 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941500.741134:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941500.741139:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941500.741149:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941500.741154:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941500.741155:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941500.741166:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941500.741168:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d09350540. 00000100:00100000:3.0:1773941500.741193:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06505180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291477376:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941500.741193:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:2.0:1773941500.741197:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941500.741204:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00000001:2.0:1773941500.741207:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 02000000:00000001:3.0:1773941500.741212:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00000001:2.0:1773941500.741213:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 02000000:00000001:3.0:1773941500.741218:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941500.741218:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941500.741223:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089dfa00 refcount=4 02000000:00000001:3.0:1773941500.741225:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:2.0:1773941500.741230:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941500.741236:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 02000000:00000010:3.0:1773941500.741237:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03522c00. 02000000:00000001:3.0:1773941500.741244:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941500.741248:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28e2 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941500.741259:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941500.741273:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00010000:00000001:2.0:1773941500.741274:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00010000:00010000:2.0:1773941500.741283:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28e2 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:3.0:1773941500.741286:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00010000:00000001:2.0:1773941500.741298:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000200:3.0:1773941500.741299:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291477376, portal 25 00000100:00000001:3.0:1773941500.741305:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00010000:2.0:1773941500.741305:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28e2 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941500.741311:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196578688 : -113743512972928 : ffff988d06505180) 00000100:00000040:3.0:1773941500.741324:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06505180 x1860112291477376/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941510 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1773941500.741326:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941500.741338:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:2.0:1773941500.741341:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28e2 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941500.741381:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:2.0:1773941500.741384:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000040:3.0:1773941500.741390:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00000001:2.0:1773941500.741391:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1773941500.741397:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182accc0. 00010000:00000001:2.0:1773941500.741398:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:2.0:1773941500.741405:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28e2 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:3.0:1773941500.741408:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291477376, offset 0 00010000:00010000:2.0:1773941500.741425:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d089dfa00 already unused, calling callback (ffffffffc12b4c20) 00000400:00000200:3.0:1773941500.741427:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 10000000:00000001:2.0:1773941500.741437:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:2.0:1773941500.741442:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28e2 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941500.741446:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:2.0:1773941500.741457:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000400:00000200:3.0:1773941500.741462:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:2.0:1773941500.741462:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:2.0:1773941500.741469:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941500.741473:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089dfa00 refcount=5 00000400:00000200:3.0:1773941500.741474:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:2.0:1773941500.741480:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00000400:00000200:3.0:1773941500.741483:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00000001:2.0:1773941500.741488:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00000400:00000200:3.0:1773941500.741494:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:2.0:1773941500.741498:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:3.0:1773941500.741502:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00010000:2.0:1773941500.741506:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28e2 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941500.741513:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[6, 7] 00000400:00000200:3.0:1773941500.741531:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 10000000:00000001:2.0:1773941500.741533:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:3.0:1773941500.741540:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [118:118:256:4294967295] 192.168.204.114@tcp LPNI seq info [8:8:8:4294967295] 10000000:00010000:2.0:1773941500.741541:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28e2 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941500.741564:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 10000000:01000000:2.0:1773941500.741569:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:2.0:1773941500.741575:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:2.0:1773941500.741580:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941500.741589:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00010000:00010000:2.0:1773941500.741591:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:2.0:1773941500.741600:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000800:00000200:3.0:1773941500.741605:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:3.0:1773941500.741616:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74500. 00010000:00000001:2.0:1773941500.741616:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1773941500.741620:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1773941500.741624:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941500.741627:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941500.741636:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000020:00000040:2.0:1773941500.741638:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d089dfa00 with handle 0x8e86767622e28e2 from hash 00010000:00000001:2.0:1773941500.741645:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1773941500.741648:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941500.741650:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000200:3.0:1773941500.741655:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00010000:00000001:2.0:1773941500.741656:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1773941500.741659:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000800:00000200:3.0:1773941500.741669:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74500 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:2.0:1773941500.741670:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:2.0:1773941500.741682:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941500.741684:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941500.741686:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941500.741689:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941500.741692:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28e2 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941500.741696:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941500.741709:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:3.0:1773941500.741710:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:2.0:1773941500.741714:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:3.0:1773941500.741717:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294892337 00010000:00000001:2.0:1773941500.741719:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:2.0:1773941500.741725:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:2.0:1773941500.741727:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941500.741729:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941500.741731:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941500.741733:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294892327 00010000:00010000:2.0:1773941500.741735:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28e2 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941500.741744:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294892337 00010000:00000040:2.0:1773941500.741748:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d071ee100 count: 0 00000100:00000001:3.0:1773941500.741751:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294892327 00000100:00000001:3.0:1773941500.741760:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941500.741765:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941500.741772:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941500.741776:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941500.741781:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941500.741783:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1773941500.741788:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941500.741795:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06507480 x1860112291477248/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941516 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941500.741800:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941500.741805:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941500.741810:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1773941500.741811:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941500.741815:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1773941500.741817:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1773941500.741819:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941500.741821:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1773941500.741823:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1773941500.741828:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74500. 02000000:00000010:2.0:1773941500.741831:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a4800. 00000400:00000200:0.0:1773941500.741836:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:2.0:1773941500.741838:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941500.741843:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000400:00000200:0.0:1773941500.741847:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000020:00000040:2.0:1773941500.741849:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000400:00000200:0.0:1773941500.741853:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182accc0 00000020:00000001:2.0:1773941500.741859:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 00000400:00000010:0.0:1773941500.741859:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182accc0. 00000100:00000001:0.0:1773941500.741867:0:2393:0:(events.c:34:request_out_callback()) Process entered 02000000:00000010:2.0:1773941500.741868:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d0381f200. 00000100:00000200:0.0:1773941500.741874:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06505180 x1860112291477376/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941510 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1773941500.741878:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941500.741883:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1773941500.741890:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000010:2.0:1773941500.741891:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06507480. 00000100:00000040:0.0:1773941500.741896:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06505180 x1860112291477376/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941510 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941500.741900:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941500.741906:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941500.741907:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941500.741914:0:2393:0:(events.c:68:request_out_callback()) Process leaving 10000000:00000001:2.0:1773941500.741915:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:2.0:1773941500.741922:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:2.0:1773941500.741926:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:2.0:1773941500.741931:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d05437dc0. 00000020:00000010:2.0:1773941500.741936:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d0381f200. 00000020:00000010:2.0:1773941500.741942:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d0381e000. 00000040:00000010:2.0:1773941500.741947:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d0381e800. 00000020:00000010:2.0:1773941500.741952:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941500.741998:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d03356200. 80000000:00000010:2.0:1773941500.742005:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d08bbb420. 00040000:00000010:2.0:1773941500.742011:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d08a4f400. 20000000:00000010:2.0:1773941500.742016:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d08a4fe00. 10000000:00000040:2.0:1773941500.742022:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:2.0:1773941500.742025:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:2.0:1773941500.742030:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d0381e000. 00000040:00000010:2.0:1773941500.742036:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d0381e800. 00000020:00000010:2.0:1773941500.742039:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941500.742045:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d03356200. 80000000:00000010:2.0:1773941500.742049:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d08bbb420. 00040000:00000010:2.0:1773941500.742053:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d08a4f400. 20000000:00000010:2.0:1773941500.742057:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d08a4fe00. 00000020:00000010:2.0:1773941500.742061:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0381f200. 10000000:00000010:2.0:1773941500.742064:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d05437dc0. 10000000:01000000:2.0:1773941500.742067:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:2.0:1773941500.742077:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:2.0:1773941500.742080:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:2.0:1773941500.744360:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:2.0:1773941500.744363:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:2.0:1773941500.744366:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941500.744369:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00010000:00000010:2.1:1773941500.746998:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d071ee100. 00010000:00000010:2.1:1773941500.747007:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d089dfa00. 00000800:00000001:0.0:1773941500.747127:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941500.747142:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941500.747145:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941500.747148:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941500.747157:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941500.747193:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941500.747203:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941500.747209:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941500.747215:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941500.747219:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324603780 00000400:00000200:0.0:1773941500.747225:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941500.747231:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xb01 [1] + 0 00000800:00000001:0.0:1773941500.747239:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941500.747258:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941500.747261:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941500.747264:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941500.747270:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941500.747276:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06505180 x1860112291477376/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941510 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941500.747284:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d06505180 x1860112291477376/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941510 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941500.747300:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941500.747308:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941500.747311:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941500.747325:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941500.747330:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941500.747335:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941500.747337:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941500.747340:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941500.747344:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941500.747346:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941500.747348:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941500.747353:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06505180 x1860112291477376/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941510 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941500.747361:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06505180 x1860112291477376/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941510 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941500.747366:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941500.747367:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941500.747369:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941500.747371:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941500.747373:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941500.747375:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941500.747376:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941500.747381:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=1 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941500.747384:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941500.747386:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941500.747387:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941500.747389:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941500.747391:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941500.747393:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941500.747396:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06505180 x1860112291477376/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941510 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941500.747410:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941500.747413:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941500.747414:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941500.747416:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941500.747421:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941500.747423:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941500.747424:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941500.747426:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941500.747428:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941500.747429:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941500.747431:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941500.747433:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941500 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941500.747435:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941500.747440:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941500.747442:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941500.747444:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941500.747445:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941500.747446:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941500.747449:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941500.747451:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941500.747453:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941500.747456:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941500.747467:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941500.747469:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941500.747672:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941500.747676:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941500.747682:0:14714:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000040:2.0:1773941500.747682:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06505180 x1860112291477376/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941510 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1773941500.747692:0:14714:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:2.0:1773941500.747693:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06505180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291477376:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00100000:3.0:1773941500.747697:0:14714:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:3.0:1773941500.747700:0:14714:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:2.0:1773941500.747700:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 10000000:01000000:3.0:1773941500.747701:0:14714:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000001:2.0:1773941500.747705:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000001:00000001:3.0:1773941500.747706:0:14714:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941500.747709:0:14714:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000040:2.0:1773941500.747710:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06505180 x1860112291477376/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941510 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941500.747718:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:3.0:1773941500.747719:0:14714:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941500.747720:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:3.0:1773941500.747722:0:14714:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000010:2.0:1773941500.747724:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03522c00. 00000001:00000001:3.0:1773941500.747725:0:14714:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941500.747729:0:14714:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941500.747730:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 10000000:00000001:3.0:1773941500.747731:0:14714:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941500.747733:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:3.0:1773941500.747734:0:14714:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000040:2.0:1773941500.747736:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941500.747739:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00080000:3.0:1773941500.747740:0:14714:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 02000000:00000010:2.0:1773941500.747741:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d035a5c00. 00000100:00000001:3.0:1773941500.747744:0:14714:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 02000000:00000001:2.0:1773941500.747745:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1773941500.747747:0:14714:0:(recover.c:147:ptlrpc_resend()) Process entered 02000000:00000001:2.0:1773941500.747748:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1773941500.747750:0:14714:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941500.747751:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06505180. 00000100:00080000:3.0:1773941500.747753:0:14714:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941500.747755:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941500.747757:0:14714:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:2.0:1773941500.747758:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:01000000:3.0:1773941500.747760:0:14714:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000100:00000001:2.0:1773941500.747761:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:00000040:3.0:1773941500.747763:0:14714:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00000001:2.0:1773941500.747768:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941500.747771:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941500.747772:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941500.747774:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1773941500.747782:0:14714:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:3.0:1773941500.747785:0:14714:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941500.747788:0:14714:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:3.0:1773941500.747793:0:14714:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:3.0:1773941500.750394:0:14714:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941500.750400:0:14714:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941500.750402:0:14714:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941500.750404:0:14714:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941500.750405:0:14714:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941500.923077:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941500.923099:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941500.923103:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1773941501.947174:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1773941501.947181:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941501.947186:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941502.011087:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294892337, at 4294892352 00000100:00000040:0.0:1773941502.011087:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294892337, at 4294892352 00000100:00000010:1.0:1773941502.011104:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3f00. 00000100:00000010:0.0:1773941502.011106:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3de0. 10000000:00000001:0.0:1773941502.459087:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1773941502.459095:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941502.459099:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941502.459102:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941502.459103:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941502.459105:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1773941502.459108:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:0.0:1773941502.459110:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:0.0:1773941502.459122:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:0.0:1773941502.459128:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1773941502.459130:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:0.0:1773941502.459142:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10837b80. 00000020:00000040:0.0:1773941502.459189:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941502.459194:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941502.459197:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941502.459203:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941502.459210:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065c4a00. 00000100:00000001:0.0:1773941502.459217:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941502.459222:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1773941502.459225:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1773941502.459233:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d21ec1400. 00010000:00000001:0.0:1773941502.459238:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1773941502.459242:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d00030c00. 00000020:00000001:0.0:1773941502.459246:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:0.0:1773941502.459249:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d00030c00 with handle 0x8e86767622e28e9 to hash 00000020:00000001:0.0:1773941502.459251:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:0.0:1773941502.459253:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330090851328 : -113743618700288 : ffff988d00030c00) 00010000:00000001:0.0:1773941502.459256:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330090851328 : -113743618700288 : ffff988d00030c00) 00010000:00000001:0.0:1773941502.459260:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941502.459264:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941502.459268:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e28e9 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941502.459279:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e28e9 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941502.459292:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e28e9 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941502.459298:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1773941502.459300:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941502.459307:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb6840. 00000100:00000001:0.0:1773941502.459309:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330139633728 : -113743569917888 : ffff988d02eb6840) 00000100:00000001:0.0:1773941502.459312:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941502.459314:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367703936 : -113743341847680 : ffff988d10837b80) 00000100:00000001:0.0:1773941502.459319:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941502.459340:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941502.459343:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941502.459345:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941502.459356:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10837b80 x1860112291477504/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941502.459366:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941502.459369:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941502.459372:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941502.459376:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941502.459378:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941502.459382:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10837b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291477504:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941502.459387:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941502.459391:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941502.459394:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941502.459396:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941502.459401:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d02ce7400. 02000000:00000001:0.0:1773941502.459403:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941502.459410:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941502.459414:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0014baa8. 00000400:00000010:0.0:1773941502.459421:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833908. 00000100:00000200:0.0:1773941502.459428:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291477504, portal 25 00000100:00000001:0.0:1773941502.459430:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941502.459431:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367703936 : -113743341847680 : ffff988d10837b80) 00000100:00000040:0.0:1773941502.459437:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10837b80 x1860112291477504/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941518 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941502.459444:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941502.459448:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941502.459450:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833dd0. 00000100:00000200:0.0:1773941502.459453:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291477504, offset 0 00000400:00000200:0.0:1773941502.459460:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941502.459470:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941502.459478:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941502.459484:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941502.459495:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941502.459499:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941502.459502:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941502.459505:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[8, 7] 00000400:00000200:0.0:1773941502.459509:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:0.0:1773941502.459514:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [119:119:256:4294967295] 192.168.204.124@tcp LPNI seq info [9:9:8:4294967295] 00000400:00000200:0.0:1773941502.459518:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941502.459526:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:0.0:1773941502.459531:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:0.0:1773941502.459535:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a20bd00. 00000800:00000200:0.0:1773941502.459541:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:0.0:1773941502.459547:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:0.0:1773941502.459551:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a20bd00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1773941502.459570:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941502.459573:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941502.459575:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294894055 00000100:00000010:0.0:1773941502.459578:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580b40. 00000100:00000001:0.0:1773941502.459584:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1773941502.459589:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294894055 00000100:00000010:0.0:1773941502.459591:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d065806c0. 00000100:00000001:0.0:1773941502.459593:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1773941502.459596:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941502.459598:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941502.459600:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941502.459601:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1773941502.459603:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6840 going to sleep for 11 seconds 00000100:00000001:0.0:1773941502.459605:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941502.459607:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941502.459755:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941502.459760:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a20bd00. 00000400:00000200:1.0:1773941502.459765:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941502.459770:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941502.459775:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833dd0 00000400:00000010:1.0:1773941502.459777:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833dd0. 00000100:00000001:1.0:1773941502.459780:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941502.459784:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10837b80 x1860112291477504/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941518 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941502.459792:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941502.459794:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d10837b80 x1860112291477504/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941518 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941502.459800:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941502.459801:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941502.461010:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941502.461026:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941502.461029:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941502.461041:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941502.461047:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941502.461051:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941502.461055:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941502.461058:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941502.461061:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941502.461065:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324603800 00000400:00000200:0.0:1773941502.461069:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941502.461073:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xb09 [1] + 224 00000800:00000001:0.0:1773941502.461079:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941502.461086:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941502.461087:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941502.461091:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941502.461095:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941502.461100:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10837b80 x1860112291477504/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941518 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941502.461108:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d10837b80 x1860112291477504/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941518 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941502.461124:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941502.461129:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941502.461132:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941502.461174:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941502.461180:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941502.461181:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941502.461184:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0014baa8. 00000400:00000200:2.0:1773941502.461187:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833908 00000400:00000010:2.0:1773941502.461188:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833908. 00000100:00000001:2.0:1773941502.461190:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941502.461194:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10837b80 x1860112291477504/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941518 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941502.461201:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10837b80 x1860112291477504/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941518 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941502.461206:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941502.461207:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941502.461209:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941502.461211:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941502.461213:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941502.461215:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941502.461216:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941502.461221:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=3 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941502.461224:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941502.461226:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:2.0:1773941502.461229:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:2.0:1773941502.464634:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941502.464642:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:2.0:1773941502.464646:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1773941502.464651:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1773941502.464655:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1773941502.468715:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941502.468719:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941502.468721:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941502.468736:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:2.0:1773941502.468744:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d10837b80 x1860112291477504/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941518 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941502.468756:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941502.468757:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941502.468760:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941502.468763:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941502.468767:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:2.0:1773941502.468769:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941502.468771:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941502.468772:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941502.468774:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941502.468776:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941502.468778:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941502.468780:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1773941502.468782:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1773941502.468788:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941502.468793:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 223 00000400:00000200:2.0:1773941502.468801:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941502.468807:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:2.0:1773941502.468812:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941502.468815:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941502.468818:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941502.468821:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:2.0:1773941502.468825:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941502.468827:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941502.468830:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941502.468833:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941502.468835:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941502.468838:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:2.0:1773941502.468841:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941502.468844:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941502.468845:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941502.468848:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:2.0:1773941502.468858:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:2.0:1773941502.468862:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:2.0:1773941502.468865:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941502.468868:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941502.468872:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941502.468876:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941502.468879:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941502.468881:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941502.468887:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06505180. 00000020:00000040:2.0:1773941502.468891:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:2.0:1773941502.468896:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941502.468898:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941502.468901:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941502.468904:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941502.468908:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941502.468915:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d035a7c00. 00000100:00000001:2.0:1773941502.468921:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941502.468929:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d06505180 x1860112291477632/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941502.468940:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941502.468944:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941502.468950:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bc0cdaa] to pc [ptlrpcd_rcv+-1] req@ffff988d06505180 x1860112291477632/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941502.468997:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941502.469001:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:2.0:1773941502.469004:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941502.469007:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941502.469012:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941502.469014:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941502.469018:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6840 going to sleep for 1 seconds 00000100:00000001:2.0:1773941502.469022:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941502.469025:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941502.469031:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10837b80 x1860112291477504/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941518 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941502.469048:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941502.469052:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941502.469056:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941502.469056:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941502.469059:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941502.469065:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941502.469070:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941502.469071:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10837b80 x1860112291477504/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941518 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941502.469084:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06505180 x1860112291477632/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941502.469097:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10837b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291477504:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941502.469108:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941502.469113:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941502.469118:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941502.469122:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941502.469125:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941502.469125:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1773941502.469137:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06505180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291477632:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941502.469138:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941502.469163:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1773941502.469164:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:2.0:1773941502.469167:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941502.469174:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10837b80 x1860112291477504/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941518 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1773941502.469186:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1773941502.469189:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941502.469190:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941502.469198:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:2.0:1773941502.469200:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb6840. 02000000:00000010:3.0:1773941502.469205:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03522400. 02000000:00000001:3.0:1773941502.469208:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941502.469210:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:2.0:1773941502.469214:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000200:3.0:1773941502.469220:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:2.0:1773941502.469224:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000400:00000010:3.0:1773941502.469225:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00010000:00000001:2.0:1773941502.469231:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000010:3.0:1773941502.469234:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00000020:00000001:2.0:1773941502.469235:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941502.469241:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030c00 refcount=4 00000100:00000200:3.0:1773941502.469244:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291477632, portal 25 00000020:00000001:2.0:1773941502.469245:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090851328 : -113743618700288 : ffff988d00030c00) 00000100:00000001:3.0:1773941502.469248:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941502.469252:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196578688 : -113743512972928 : ffff988d06505180) 00010000:00000001:2.0:1773941502.469255:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090851328 : -113743618700288 : ffff988d00030c00) 00000100:00000040:3.0:1773941502.469263:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06505180 x1860112291477632/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941512 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:2.0:1773941502.469268:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e28e9 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941502.469270:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941502.469278:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:3.0:1773941502.469284:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acc38. 00000100:00000200:3.0:1773941502.469290:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291477632, offset 0 00010000:00000001:2.0:1773941502.469301:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000200:3.0:1773941502.469303:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00010000:2.0:1773941502.469312:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e28e9 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941502.469314:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941502.469322:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941502.469333:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941502.469339:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941502.469348:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:2.0:1773941502.469348:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000400:00000200:3.0:1773941502.469359:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00010000:2.0:1773941502.469360:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e28e9 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941502.469365:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[8, 9] 00000400:00000200:3.0:1773941502.469383:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00010000:00000001:2.0:1773941502.469392:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:3.0:1773941502.469396:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [120:120:256:4294967295] 192.168.204.114@tcp LPNI seq info [10:10:8:4294967295] 00010000:00000001:2.0:1773941502.469397:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:2.0:1773941502.469403:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e28e9 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941502.469411:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:2.0:1773941502.469425:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941502.469435:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941502.469438:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00010000:00000001:2.0:1773941502.469442:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:2.0:1773941502.469448:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e28e9 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941502.469449:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00010000:00010000:2.0:1773941502.469464:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d00030c00 already unused, calling callback (ffffffffc12b4c20) 10000000:00000001:2.0:1773941502.469467:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000800:00000010:3.0:1773941502.469470:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74f00. 10000000:00010000:2.0:1773941502.469471:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e28e9 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941502.469488:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00010000:00000001:2.0:1773941502.469489:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:2.0:1773941502.469493:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:2.0:1773941502.469495:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941502.469497:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030c00 refcount=5 00000020:00000001:2.0:1773941502.469505:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090851328 : -113743618700288 : ffff988d00030c00) 00010000:00000001:2.0:1773941502.469508:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090851328 : -113743618700288 : ffff988d00030c00) 00000800:00000200:3.0:1773941502.469515:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00010000:00000001:2.0:1773941502.469516:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000800:00000200:3.0:1773941502.469522:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74f00 type 1, nob 616 niov 1 nkiov 1 00010000:00010000:2.0:1773941502.469524:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e28e9 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:2.0:1773941502.469535:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000001:3.0:1773941502.469544:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 10000000:00010000:2.0:1773941502.469546:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e28e9 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941502.469554:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:3.0:1773941502.469562:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294894065 10000000:01000000:2.0:1773941502.469573:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000100:00000001:3.0:1773941502.469577:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294894055 10000000:00000040:2.0:1773941502.469578:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:2.0:1773941502.469581:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941502.469583:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294894065 00010000:00010000:2.0:1773941502.469587:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000100:00000001:3.0:1773941502.469591:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294894055 00010000:00000001:2.0:1773941502.469591:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000100:00000001:3.0:1773941502.469598:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:2.0:1773941502.469598:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1773941502.469605:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000100:00000001:3.0:1773941502.469607:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941502.469617:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:3.0:1773941502.469618:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941502.469626:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941502.469627:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1773941502.469643:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d00030c00 with handle 0x8e86767622e28e9 from hash 00000100:00000001:3.0:1773941502.469648:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:2.0:1773941502.469649:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000100:00000001:3.0:1773941502.469651:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00010000:00000001:2.0:1773941502.469653:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941502.469659:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:2.0:1773941502.469660:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941502.469668:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000100:00000001:3.0:1773941502.469669:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941502.469678:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941502.469679:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000800:00000200:1.0:1773941502.469685:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1773941502.469689:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941502.469691:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000800:00000010:1.0:1773941502.469692:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74f00. 00010000:00000001:2.0:1773941502.469704:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941502.469707:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:1.0:1773941502.469708:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000001:2.0:1773941502.469709:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941502.469713:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941502.469718:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e28e9 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:1.0:1773941502.469724:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00010000:00000001:2.0:1773941502.469725:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941502.469728:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941502.469731:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:2.0:1773941502.469733:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:2.0:1773941502.469734:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941502.469735:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:1.0:1773941502.469735:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acc38 00010000:00000001:2.0:1773941502.469737:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1773941502.469742:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e28e9 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:1.0:1773941502.469742:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acc38. 00010000:00000040:2.0:1773941502.469752:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d21ec1400 count: 0 00000100:00000001:1.0:1773941502.469752:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941502.469760:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06505180 x1860112291477632/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941512 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1773941502.469769:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941502.469772:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941502.469773:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941502.469776:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941502.469781:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10837b80 x1860112291477504/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941518 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941502.469789:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000040:1.0:1773941502.469791:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06505180 x1860112291477632/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941512 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1773941502.469792:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941502.469796:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d02ce7400. 02000000:00000001:2.0:1773941502.469801:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941502.469804:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941502.469807:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941502.469811:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941502.469812:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941502.469813:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065c4a00. 00000100:00000001:1.0:1773941502.469818:0:2393:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1773941502.469820:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941502.469822:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941502.469824:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10837b80. 00000100:00000001:2.0:1773941502.469827:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941502.469828:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:2.0:1773941502.469830:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:2.0:1773941502.469832:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:2.0:1773941502.469835:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:2.0:1773941502.469838:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d05437dc0. 00000020:00000010:2.0:1773941502.469841:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d08a4fe00. 00000020:00000010:2.0:1773941502.469845:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d08a4f400. 00000040:00000010:2.0:1773941502.469850:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d03356200. 00000020:00000010:2.0:1773941502.469853:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941502.469858:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d0381f200. 80000000:00000010:2.0:1773941502.469862:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d08bbb420. 00040000:00000010:2.0:1773941502.469866:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d0381e800. 20000000:00000010:2.0:1773941502.469869:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d0381e000. 10000000:00000040:2.0:1773941502.469872:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:2.0:1773941502.469874:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:2.0:1773941502.469878:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d08a4f400. 00000040:00000010:2.0:1773941502.469882:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d03356200. 00000020:00000010:2.0:1773941502.469888:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941502.469891:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d0381f200. 80000000:00000010:2.0:1773941502.469893:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d08bbb420. 00040000:00000010:2.0:1773941502.469895:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d0381e800. 20000000:00000010:2.0:1773941502.469897:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d0381e000. 00000020:00000010:2.0:1773941502.469899:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d08a4fe00. 10000000:00000010:2.0:1773941502.469901:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d05437dc0. 10000000:01000000:2.0:1773941502.469903:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:2.0:1773941502.469905:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:2.0:1773941502.469907:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:2.0:1773941502.472106:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:2.0:1773941502.472109:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:2.0:1773941502.472111:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941502.472114:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941502.473646:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941502.473658:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941502.473661:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941502.473663:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941502.473668:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941502.473673:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941502.473678:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941502.473682:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941502.473687:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941502.473690:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324603880 00000400:00000200:0.0:1773941502.473694:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941502.473704:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xb19 [1] + 0 00000800:00000001:0.0:1773941502.473710:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941502.473718:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941502.473720:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941502.473722:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941502.473726:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941502.473730:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06505180 x1860112291477632/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941512 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941502.473738:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d06505180 x1860112291477632/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941512 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941502.473754:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941502.473760:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941502.473762:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941502.473796:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941502.473801:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941502.473807:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941502.473809:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941502.473812:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941502.473819:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941502.473822:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941502.473826:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941502.473835:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06505180 x1860112291477632/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941512 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941502.473846:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06505180 x1860112291477632/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941512 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941502.473855:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941502.473857:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941502.473859:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941502.473872:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941502.473875:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941502.473878:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941502.473882:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941502.473891:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=3 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941502.473895:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941502.473898:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941502.473900:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941502.473904:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941502.473906:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941502.473908:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941502.473913:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06505180 x1860112291477632/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941512 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941502.473918:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941502.473920:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941502.473922:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941502.473924:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941502.473931:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941502.473933:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941502.473935:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941502.473937:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941502.473939:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941502.473941:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941502.473943:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941502.473946:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941502 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941502.473948:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941502.473954:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941502.473956:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941502.473997:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941502.474000:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941502.474003:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941502.474006:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941502.474008:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941502.474011:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941502.474021:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941502.474024:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941502.474027:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941502.474250:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941502.474257:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941502.474263:0:14715:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000040:2.0:1773941502.474270:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06505180 x1860112291477632/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941512 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1773941502.474277:0:14715:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:3.0:1773941502.474285:0:14715:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00100000:2.0:1773941502.474301:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06505180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291477632:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941502.474314:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941502.474319:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941502.474325:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06505180 x1860112291477632/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941512 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941502.474329:0:14715:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941502.474334:0:14715:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000001:2.0:1773941502.474340:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:3.0:1773941502.474341:0:14715:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:2.0:1773941502.474345:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:3.0:1773941502.474348:0:14715:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000010:2.0:1773941502.474351:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03522400. 00000001:00000001:3.0:1773941502.474355:0:14715:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941502.474358:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:3.0:1773941502.474360:0:14715:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:2.0:1773941502.474363:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:3.0:1773941502.474364:0:14715:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000040:2.0:1773941502.474367:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000001:00000001:3.0:1773941502.474369:0:14715:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941502.474375:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 10000000:00000001:3.0:1773941502.474376:0:14715:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941502.474381:0:14715:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000010:2.0:1773941502.474381:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d035a7c00. 02000000:00000001:2.0:1773941502.474389:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00080000:3.0:1773941502.474390:0:14715:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:3.0:1773941502.474395:0:14715:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:3.0:1773941502.474400:0:14715:0:(recover.c:147:ptlrpc_resend()) Process entered 02000000:00000001:2.0:1773941502.474402:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1773941502.474405:0:14715:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941502.474407:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06505180. 00000100:00080000:3.0:1773941502.474413:0:14715:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941502.474415:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941502.474417:0:14715:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941502.474423:0:14715:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000100:00000001:2.0:1773941502.474424:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000040:3.0:1773941502.474428:0:14715:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00000001:2.0:1773941502.474432:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941502.474441:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941502.474443:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941502.474446:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941502.474448:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1773941502.474453:0:14715:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:3.0:1773941502.474456:0:14715:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941502.474457:0:14715:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:3.0:1773941502.474461:0:14715:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:3.0:1773941502.476668:0:14715:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941502.476671:0:14715:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941502.476673:0:14715:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941502.476676:0:14715:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941502.476677:0:14715:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:2.1:1773941502.481007:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d21ec1400. 00010000:00000010:2.1:1773941502.481018:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d00030c00. 00000020:00000001:3.0:1773941502.971150:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941502.971160:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941502.971189:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941503.739082:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294894065, at 4294894080 00000100:00000040:0.0:1773941503.739093:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294894065, at 4294894080 00000100:00000010:1.0:1773941503.739097:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d065806c0. 00000100:00000010:0.0:1773941503.739112:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580b40. 10000000:00000001:0.0:1773941503.867107:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1773941503.867116:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941503.867131:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941503.867134:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941503.867135:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941503.867137:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1773941503.867140:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:0.0:1773941503.867142:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:0.0:1773941503.867144:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:0.0:1773941503.867146:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1773941503.867148:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:0.0:1773941503.867162:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10835880. 00000020:00000040:0.0:1773941503.867167:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941503.867172:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941503.867176:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941503.867182:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941503.867191:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065c5e00. 00000100:00000001:0.0:1773941503.867198:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941503.867221:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1773941503.867225:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1773941503.867234:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d21ec1400. 00010000:00000001:0.0:1773941503.867241:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1773941503.867244:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d00030400. 00000020:00000001:0.0:1773941503.867248:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:0.0:1773941503.867251:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d00030400 with handle 0x8e86767622e28f0 to hash 00000020:00000001:0.0:1773941503.867252:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:0.0:1773941503.867254:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00010000:00000001:0.0:1773941503.867256:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00010000:00000001:0.0:1773941503.867260:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941503.867263:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941503.867267:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e28f0 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941503.867276:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e28f0 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941503.867287:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e28f0 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941503.867292:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1773941503.867294:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941503.867301:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb66c0. 00000100:00000001:0.0:1773941503.867302:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330139633344 : -113743569918272 : ffff988d02eb66c0) 00000100:00000001:0.0:1773941503.867305:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941503.867306:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367694976 : -113743341856640 : ffff988d10835880) 00000100:00000001:0.0:1773941503.867311:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941503.867316:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941503.867319:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941503.867320:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941503.867331:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10835880 x1860112291477760/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941503.867339:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941503.867344:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941503.867348:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941503.867350:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941503.867352:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941503.867355:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10835880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291477760:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941503.867359:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941503.867373:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941503.867376:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941503.867383:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941503.867389:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d02ce7000. 02000000:00000001:0.0:1773941503.867392:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941503.867401:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941503.867407:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0014b160. 00000400:00000010:0.0:1773941503.867416:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833660. 00000100:00000200:0.0:1773941503.867425:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291477760, portal 25 00000100:00000001:0.0:1773941503.867427:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941503.867429:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367694976 : -113743341856640 : ffff988d10835880) 00000100:00000040:0.0:1773941503.867445:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10835880 x1860112291477760/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941519 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941503.867454:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941503.867457:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941503.867461:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d108337f8. 00000100:00000200:0.0:1773941503.867465:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291477760, offset 0 00000400:00000200:0.0:1773941503.867473:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941503.867484:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941503.867492:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941503.867498:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941503.867502:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941503.867506:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941503.867509:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941503.867511:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[10, 9] 00000400:00000200:0.0:1773941503.867513:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:0.0:1773941503.867517:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [121:121:256:4294967295] 192.168.204.124@tcp LPNI seq info [11:11:8:4294967295] 00000400:00000200:0.0:1773941503.867525:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941503.867536:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:0.0:1773941503.867541:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:0.0:1773941503.867546:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a20bd00. 00000800:00000200:0.0:1773941503.867553:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:0.0:1773941503.867561:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:0.0:1773941503.867565:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a20bd00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1773941503.867584:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941503.867588:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941503.867590:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294895463 00000100:00000010:0.0:1773941503.867594:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580b40. 00000100:00000001:0.0:1773941503.867602:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1773941503.867612:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294895463 00000100:00000010:0.0:1773941503.867616:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d065805a0. 00000100:00000001:0.0:1773941503.867620:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1773941503.867632:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941503.867635:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941503.867638:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941503.867640:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1773941503.867644:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb66c0 going to sleep for 11 seconds 00000100:00000001:0.0:1773941503.867648:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941503.867651:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941503.867704:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941503.867709:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a20bd00. 00000400:00000200:1.0:1773941503.867715:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941503.867720:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941503.867726:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d108337f8 00000400:00000010:1.0:1773941503.867728:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d108337f8. 00000100:00000001:1.0:1773941503.867732:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941503.867736:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10835880 x1860112291477760/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941519 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941503.867745:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941503.867748:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d10835880 x1860112291477760/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941519 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941503.867754:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941503.867755:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941503.868957:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941503.868987:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941503.868991:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941503.868994:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941503.869000:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941503.869006:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941503.869012:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941503.869016:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941503.869021:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941503.869027:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324603900 00000400:00000200:0.0:1773941503.869038:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941503.869044:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xb21 [1] + 224 00000800:00000001:0.0:1773941503.869053:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941503.869063:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941503.869065:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941503.869069:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941503.869075:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941503.869080:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10835880 x1860112291477760/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941519 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941503.869091:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d10835880 x1860112291477760/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941519 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941503.869115:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941503.869139:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941503.869143:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941503.869172:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941503.869181:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941503.869183:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941503.869186:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0014b160. 00000400:00000200:2.0:1773941503.869191:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833660 00000400:00000010:2.0:1773941503.869194:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833660. 00000100:00000001:2.0:1773941503.869197:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941503.869203:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10835880 x1860112291477760/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941519 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941503.869213:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10835880 x1860112291477760/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941519 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941503.869221:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941503.869224:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941503.869227:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941503.869230:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941503.869233:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941503.869236:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941503.869239:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941503.869246:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=4 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941503.869260:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941503.869264:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:2.0:1773941503.869269:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:2.0:1773941503.869272:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941503.869275:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:2.0:1773941503.869278:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1773941503.869282:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1773941503.869293:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1773941503.869296:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941503.869298:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941503.869299:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941503.869301:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:2.0:1773941503.869305:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d10835880 x1860112291477760/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941519 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941503.869311:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941503.869312:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941503.869314:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941503.869317:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941503.869319:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:2.0:1773941503.869320:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941503.869321:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941503.869322:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941503.869323:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941503.869324:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941503.869326:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941503.869328:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1773941503.869329:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1773941503.869332:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941503.869333:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 225 00000400:00000200:2.0:1773941503.869337:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941503.869347:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:2.0:1773941503.869352:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941503.869355:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941503.869357:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941503.869360:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:2.0:1773941503.869363:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941503.869365:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941503.869368:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941503.869371:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941503.869374:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941503.869383:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:2.0:1773941503.869387:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941503.869389:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941503.869391:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941503.869394:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:2.0:1773941503.869398:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:2.0:1773941503.869401:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:2.0:1773941503.869405:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941503.869408:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941503.869411:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941503.869414:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941503.869417:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941503.869419:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941503.869423:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06505180. 00000020:00000040:2.0:1773941503.869425:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:2.0:1773941503.869428:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941503.869429:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941503.869431:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941503.869432:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941503.869434:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941503.869438:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d035a4800. 00000100:00000001:2.0:1773941503.869442:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941503.869459:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d06505180 x1860112291477888/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941503.869469:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941503.869472:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941503.869477:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007bc0cdaa] to pc [ptlrpcd_rcv+-1] req@ffff988d06505180 x1860112291477888/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941503.869501:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941503.869504:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:2.0:1773941503.869505:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941503.869508:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941503.869515:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941503.869517:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941503.869518:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb66c0 going to sleep for 1 seconds 00000100:00000001:2.0:1773941503.869520:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941503.869522:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941503.869527:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10835880 x1860112291477760/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941519 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941503.869530:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941503.869535:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941503.869536:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941503.869538:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1773941503.869540:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941503.869543:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:3.0:1773941503.869547:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06505180 x1860112291477888/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941503.869547:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941503.869553:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10835880 x1860112291477760/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941519 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941503.869565:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941503.869569:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941503.869571:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10835880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291477760:192.168.204.114@tcp:101:ll_cfg_requeue.0 02000000:00000001:3.0:1773941503.869572:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941503.869576:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941503.869578:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1773941503.869580:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941503.869582:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00100000:3.0:1773941503.869586:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06505180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291477888:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941503.869587:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:2.0:1773941503.869590:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1773941503.869592:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941503.869596:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:2.0:1773941503.869597:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10835880 x1860112291477760/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941519 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1773941503.869598:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941503.869603:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941503.869608:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03523800. 00000100:00000001:2.0:1773941503.869608:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941503.869611:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb66c0. 02000000:00000001:3.0:1773941503.869612:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941503.869616:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000400:00000200:3.0:1773941503.869619:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941503.869620:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000010:3.0:1773941503.869623:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00010000:00000001:2.0:1773941503.869624:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:2.0:1773941503.869629:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000010:3.0:1773941503.869630:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00000020:00000001:2.0:1773941503.869631:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941503.869635:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030400 refcount=4 00000100:00000200:3.0:1773941503.869636:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291477888, portal 25 00000020:00000001:2.0:1773941503.869637:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00000100:00000001:3.0:1773941503.869640:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00000001:2.0:1773941503.869641:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00000100:00000001:3.0:1773941503.869643:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196578688 : -113743512972928 : ffff988d06505180) 00010000:00010000:2.0:1773941503.869646:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e28f0 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:3.0:1773941503.869649:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06505180 x1860112291477888/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941513 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941503.869670:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:2.0:1773941503.869671:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000040:3.0:1773941503.869677:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00010000:2.0:1773941503.869679:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e28f0 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:3.0:1773941503.869680:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac330. 00000100:00000200:3.0:1773941503.869686:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291477888, offset 0 00010000:00000001:2.0:1773941503.869688:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:2.0:1773941503.869691:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e28f0 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941503.869692:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941503.869701:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:2.0:1773941503.869701:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941503.869704:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:3.0:1773941503.869706:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00010000:2.0:1773941503.869708:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e28f0 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941503.869713:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941503.869716:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941503.869720:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:2.0:1773941503.869720:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:3.0:1773941503.869725:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00010000:2.0:1773941503.869725:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e28f0 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941503.869731:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941503.869736:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00010000:2.0:1773941503.869738:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d00030400 already unused, calling callback (ffffffffc12b4c20) 00000400:00000200:3.0:1773941503.869739:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[10, 11] 10000000:00000001:2.0:1773941503.869741:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:2.0:1773941503.869745:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e28f0 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941503.869746:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941503.869753:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [122:122:256:4294967295] 192.168.204.114@tcp LPNI seq info [12:12:8:4294967295] 00010000:00000001:2.0:1773941503.869760:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000400:00000200:3.0:1773941503.869761:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:2.0:1773941503.869763:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:2.0:1773941503.869766:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941503.869770:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030400 refcount=5 00000020:00000001:2.0:1773941503.869774:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00000400:00000200:3.0:1773941503.869775:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00010000:00000001:2.0:1773941503.869779:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00000800:00000200:3.0:1773941503.869784:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00010000:00000001:2.0:1773941503.869786:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000800:00000010:3.0:1773941503.869789:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74800. 00000800:00000200:3.0:1773941503.869794:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00010000:00010000:2.0:1773941503.869799:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e28f0 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941503.869804:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:3.0:1773941503.869809:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74800 type 1, nob 616 niov 1 nkiov 1 10000000:00000001:2.0:1773941503.869818:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:2.0:1773941503.869821:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e28f0 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941503.869823:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941503.869833:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 10000000:01000000:2.0:1773941503.869834:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000100:00000001:3.0:1773941503.869836:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294895465 10000000:00000040:2.0:1773941503.869840:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:2.0:1773941503.869845:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941503.869846:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294895463 00010000:00010000:2.0:1773941503.869850:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000100:00000001:3.0:1773941503.869853:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294895465 00010000:00000001:2.0:1773941503.869855:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000100:00000001:3.0:1773941503.869859:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294895463 00010000:00000001:2.0:1773941503.869861:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000100:00000001:3.0:1773941503.869866:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:2.0:1773941503.869866:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1773941503.869870:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:3.0:1773941503.869872:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941503.869873:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941503.869877:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941503.869878:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d00030400 with handle 0x8e86767622e28f0 from hash 00010000:00000001:2.0:1773941503.869883:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000100:00000001:3.0:1773941503.869885:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1773941503.869886:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941503.869888:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941503.869891:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1773941503.869893:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000800:00000200:1.0:1773941503.869894:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00000001:2.0:1773941503.869895:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000100:00000001:3.0:1773941503.869898:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:2.0:1773941503.869900:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000010:1.0:1773941503.869900:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74800. 00000100:00000001:3.0:1773941503.869901:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00010000:00000001:2.0:1773941503.869903:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:3.0:1773941503.869906:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:2.0:1773941503.869906:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941503.869907:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1773941503.869909:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 10000000:00000001:2.0:1773941503.869911:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941503.869913:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941503.869916:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e28f0 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:1.0:1773941503.869916:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:3.0:1773941503.869917:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941503.869921:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac330 00000400:00000010:1.0:1773941503.869925:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac330. 00010000:00000001:2.0:1773941503.869927:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941503.869930:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:1.0:1773941503.869931:0:2393:0:(events.c:34:request_out_callback()) Process entered 00010000:00000001:2.0:1773941503.869932:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:2.0:1773941503.869936:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:2.0:1773941503.869937:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941503.869940:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000200:1.0:1773941503.869941:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06505180 x1860112291477888/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941513 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1773941503.869947:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1773941503.869951:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e28f0 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941503.869981:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941503.869986:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06505180 x1860112291477888/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941513 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000040:2.0:1773941503.869994:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d21ec1400 count: 0 00010000:00000001:2.0:1773941503.870008:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:1.0:1773941503.870010:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941503.870012:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941503.870015:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1773941503.870016:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941503.870023:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10835880 x1860112291477760/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941519 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941503.870029:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941503.870031:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941503.870034:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d02ce7000. 02000000:00000001:2.0:1773941503.870038:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941503.870040:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941503.870042:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941503.870044:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941503.870047:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065c5e00. 02000000:00000001:2.0:1773941503.870052:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941503.870053:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941503.870055:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10835880. 00000100:00000001:2.0:1773941503.870057:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941503.870058:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:2.0:1773941503.870060:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:2.0:1773941503.870062:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:2.0:1773941503.870064:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:2.0:1773941503.870068:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d05437dc0. 00000020:00000010:2.0:1773941503.870072:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d0381e000. 00000020:00000010:2.0:1773941503.870076:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d0381e800. 00000040:00000010:2.0:1773941503.870080:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d0381f200. 00000020:00000010:2.0:1773941503.870085:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941503.870089:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d03356200. 80000000:00000010:2.0:1773941503.870093:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d08bbb420. 00040000:00000010:2.0:1773941503.870097:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d08a4fe00. 20000000:00000010:2.0:1773941503.870101:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d08a4f400. 10000000:00000040:2.0:1773941503.870109:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:2.0:1773941503.870111:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:2.0:1773941503.870116:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d0381e800. 00000040:00000010:2.0:1773941503.870135:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d0381f200. 00000020:00000010:2.0:1773941503.870138:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941503.870142:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d03356200. 80000000:00000010:2.0:1773941503.870145:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d08bbb420. 00040000:00000010:2.0:1773941503.870147:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d08a4fe00. 20000000:00000010:2.0:1773941503.870151:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d08a4f400. 00000020:00000010:2.0:1773941503.870154:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0381e000. 10000000:00000010:2.0:1773941503.870156:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d05437dc0. 10000000:01000000:2.0:1773941503.870158:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:2.0:1773941503.870161:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:2.0:1773941503.870162:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 00000800:00000001:0.0:1773941503.871374:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941503.871385:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941503.871388:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941503.871391:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941503.871396:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941503.871401:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941503.871405:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941503.871407:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941503.871410:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941503.871412:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324603980 00000400:00000200:0.0:1773941503.871415:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941503.871419:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xb31 [1] + 0 00000800:00000001:0.0:1773941503.871422:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941503.871427:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941503.871429:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941503.871431:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941503.871434:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941503.871438:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06505180 x1860112291477888/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941513 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941503.871453:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d06505180 x1860112291477888/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941513 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941503.871471:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941503.871477:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941503.871481:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941503.871556:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941503.871563:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941503.871566:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941503.871567:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941503.871569:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941503.871572:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941503.871574:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941503.871575:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941503.871579:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06505180 x1860112291477888/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941513 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941503.871586:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06505180 x1860112291477888/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941513 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941503.871591:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941503.871592:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941503.871594:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941503.871595:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941503.871596:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941503.871597:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941503.871598:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941503.871601:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=4 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941503.871603:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941503.871605:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941503.871606:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941503.871608:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941503.871610:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941503.871612:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941503.871615:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06505180 x1860112291477888/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941513 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941503.871620:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941503.871621:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941503.871622:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941503.871624:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941503.871628:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941503.871629:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941503.871630:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941503.871631:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941503.871636:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941503.871638:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941503.871670:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941503.871675:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941503 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941503.871678:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941503.871685:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941503.871686:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941503.871688:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941503.871689:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941503.871690:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941503.871693:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941503.871696:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941503.871698:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941503.871700:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941503.871703:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941503.871705:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941503.871999:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941503.872005:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941503.872022:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06505180 x1860112291477888/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941513 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941503.872023:0:14716:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:3.0:1773941503.872031:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06505180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291477888:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00080000:1.0:1773941503.872033:0:14716:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00000001:3.0:1773941503.872036:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941503.872037:0:14716:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:3.0:1773941503.872039:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1773941503.872039:0:14716:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941503.872041:0:14716:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:1.0:1773941503.872044:0:14716:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941503.872048:0:14716:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000040:3.0:1773941503.872051:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06505180 x1860112291477888/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941513 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000001:00000001:1.0:1773941503.872053:0:14716:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941503.872055:0:14716:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941503.872056:0:14716:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941503.872058:0:14716:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941503.872060:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 10000000:00000001:1.0:1773941503.872060:0:14716:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941503.872062:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:1.0:1773941503.872062:0:14716:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000010:3.0:1773941503.872064:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03523800. 00000100:00080000:1.0:1773941503.872065:0:14716:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 02000000:00000001:3.0:1773941503.872067:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:1.0:1773941503.872068:0:14716:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000020:00000001:3.0:1773941503.872069:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:1.0:1773941503.872070:0:14716:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941503.872072:0:14716:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941503.872075:0:14716:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000020:00000040:3.0:1773941503.872076:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941503.872078:0:14716:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941503.872079:0:14716:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941503.872081:0:14716:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000020:00000001:3.0:1773941503.872088:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941503.872090:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d035a4800. 00000100:00080000:1.0:1773941503.872092:0:14716:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 02000000:00000001:3.0:1773941503.872094:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 10000000:00000001:1.0:1773941503.872094:0:14716:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941503.872096:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1773941503.872096:0:14716:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000010:3.0:1773941503.872098:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06505180. 00000100:02000000:1.0:1773941503.872099:0:14716:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:3.0:1773941503.872101:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941503.872103:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941503.872103:0:14716:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941503.872106:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1773941503.872106:0:14716:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941503.872108:0:14716:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941503.872110:0:14716:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941503.872113:0:14716:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941503.872131:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941503.872133:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941503.872135:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941503.872138:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941503.873108:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:2.0:1773941503.873115:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:2.0:1773941503.873135:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941503.873140:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00010000:00000010:2.1:1773941503.882020:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d21ec1400. 00010000:00000010:2.1:1773941503.882031:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d00030400. 00000020:00000001:2.0:1773941503.995199:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941503.995206:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941503.995211:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1773941505.019158:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941505.019164:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941505.019168:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941505.147133:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294895465, at 4294895488 00000100:00000040:0.0:1773941505.147133:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294895465, at 4294895488 00000100:00000010:0.0:1773941505.147156:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580b40. 00000100:00000010:1.0:1773941505.147157:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d065805a0. 00000100:00000040:1.0:1773941505.275076:0:461:0:(pinger.c:229:ptlrpc_pinger_process_import()) c1c1ff21-5f68-4bf6-bdc4-258644b24465->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1773941505.275088:0:461:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1773941505.275122:0:461:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c2300. 00000020:00000040:1.0:1773941505.275129:0:461:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941505.275134:0:461:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941505.275137:0:461:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941505.275143:0:461:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941505.275152:0:461:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff988d02cfd800. 00000100:00000001:1.0:1773941505.275159:0:461:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941505.275172:0:461:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging c1c1ff21-5f68-4bf6-bdc4-258644b24465->MGS req@ffff988d203c2300 x1860112291478016/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941505.275182:0:461:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941505.275186:0:461:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941505.275192:0:461:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093c3ab5c] to pc [ptlrpcd_00_01+1] req@ffff988d203c2300 x1860112291478016/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941505.275221:0:461:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941505.275226:0:461:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (233) 00000100:00000001:1.0:1773941505.275238:0:2401:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1773941505.275242:0:2401:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:1.0:1773941505.275245:0:2401:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941505.275252:0:2401:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941505.275254:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941505.275255:0:2401:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941505.275259:0:2401:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941505.275260:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941505.275262:0:2401:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941505.275266:0:2401:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941505.275271:0:2401:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c2300 x1860112291478016/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941505.275277:0:2401:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941505.275280:0:2401:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941505.275283:0:2401:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941505.275286:0:2401:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941505.275288:0:2401:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941505.275290:0:2401:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2401:1860112291478016:192.168.204.114@tcp:400:kworker.0 00000100:00000001:1.0:1773941505.275326:0:2401:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941505.275332:0:2401:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941505.275335:0:2401:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941505.275337:0:2401:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941505.275341:0:2401:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff988d181fc200. 02000000:00000001:1.0:1773941505.275343:0:2401:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941505.275351:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941505.275354:0:2401:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c5d8. 00000400:00000010:1.0:1773941505.275361:0:2401:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000100:00000200:1.0:1773941505.275369:0:2401:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112291478016, portal 25 00000100:00000001:1.0:1773941505.275371:0:2401:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941505.275372:0:2401:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631463680 : -113743078087936 : ffff988d203c2300) 00000100:00000040:1.0:1773941505.275377:0:2401:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c2300 x1860112291478016/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941521 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941505.275382:0:2401:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941505.275385:0:2401:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941505.275387:0:2401:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de330. 00000100:00000200:1.0:1773941505.275390:0:2401:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1860112291478016, offset 0 00000400:00000200:1.0:1773941505.275396:0:2401:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941505.275404:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941505.275411:0:2401:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941505.275417:0:2401:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941505.275421:0:2401:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941505.275425:0:2401:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941505.275427:0:2401:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941505.275430:0:2401:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[12, 11] 00000400:00000200:1.0:1773941505.275432:0:2401:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:1.0:1773941505.275436:0:2401:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [123:123:256:4294967295] 192.168.204.124@tcp LPNI seq info [13:13:8:4294967295] 00000400:00000200:1.0:1773941505.275439:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941505.275447:0:2401:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:1.0:1773941505.275452:0:2401:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:1.0:1773941505.275459:0:2401:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfd900. 00000800:00000200:1.0:1773941505.275466:0:2401:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:1.0:1773941505.275474:0:2401:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:1.0:1773941505.275478:0:2401:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfd900 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1773941505.275492:0:2401:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941505.275495:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941505.275498:0:2401:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294896871 00000100:00000010:1.0:1773941505.275502:0:2401:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3f00. 00000100:00000001:1.0:1773941505.275510:0:2401:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1773941505.275517:0:2401:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294896871 00000100:00000010:1.0:1773941505.275519:0:2401:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3300. 00000100:00000001:1.0:1773941505.275522:0:2401:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1773941505.275525:0:2401:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941505.275527:0:2401:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941505.275530:0:2401:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941505.275531:0:2401:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941505.275614:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941505.275619:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfd900. 00000400:00000200:0.0:1773941505.275626:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941505.275633:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941505.275639:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de330 00000400:00000010:0.0:1773941505.275643:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de330. 00000100:00000001:0.0:1773941505.275647:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941505.275653:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c2300 x1860112291478016/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941521 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941505.275665:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941505.275669:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c2300 x1860112291478016/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941521 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941505.275678:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941505.275680:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941505.276792:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941505.276805:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941505.276808:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941505.276811:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941505.276816:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941505.276822:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941505.276829:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941505.276834:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941505.276846:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941505.276852:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324603a00 00000400:00000200:0.0:1773941505.276858:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941505.276863:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xb39 [1] + 224 00000800:00000001:0.0:1773941505.276871:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941505.276879:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941505.276882:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941505.276886:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941505.276891:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941505.276896:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c2300 x1860112291478016/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941521 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941505.276907:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff988d203c2300 x1860112291478016/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941521 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941505.276925:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941505.276931:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941505.276934:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941505.276954:0:2401:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941505.276957:0:2401:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941505.276976:0:2401:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941505.276978:0:2401:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941505.276980:0:2401:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:1.0:1773941505.276984:0:2401:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:1.0:1773941505.276985:0:2401:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:1.0:1773941505.276987:0:2401:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941505.276991:0:2401:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c2300 x1860112291478016/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941521 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941505.276999:0:2401:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c2300 x1860112291478016/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941521 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941505.277004:0:2401:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941505.277005:0:2401:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941505.277007:0:2401:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941505.277009:0:2401:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941505.277011:0:2401:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941505.277019:0:2401:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941505.277021:0:2401:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941505.277028:0:2401:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=6 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941505.277030:0:2401:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941505.277033:0:2401:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1773941505.277034:0:2401:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941505.277037:0:2401:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941505.277039:0:2401:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941505.277043:0:2401:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941505.277046:0:2401:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941505.281042:0:2401:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941505.281045:0:2401:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941505.281046:0:2401:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941505.281049:0:2401:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941505.281053:0:2401:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d203c2300 x1860112291478016/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941521 ref 1 fl Rpc:RNQU/200/0 rc 0/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941505.281059:0:2401:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941505.281061:0:2401:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941505.281063:0:2401:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941505.281068:0:2401:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941505.281071:0:2401:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941505.281072:0:2401:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941505.281074:0:2401:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941505.281075:0:2401:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941505.281076:0:2401:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941505.281077:0:2401:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941505.281079:0:2401:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941505.281081:0:2401:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941505.281083:0:2401:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941505.281085:0:2401:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941505.281088:0:2401:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 226 00000400:00000200:1.0:1773941505.281111:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941505.281115:0:2401:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941505.281125:0:2401:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941505.281128:0:2401:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941505.281129:0:2401:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941505.281131:0:2401:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941505.281135:0:2401:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941505.281136:0:2401:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941505.281139:0:2401:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941505.281141:0:2401:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941505.281143:0:2401:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941505.281144:0:2401:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941505.281147:0:2401:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941505.281148:0:2401:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941505.281150:0:2401:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941505.281151:0:2401:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941505.281154:0:2401:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941505.281156:0:2401:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941505.281158:0:2401:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941505.281161:0:2401:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941505.281163:0:2401:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941505.281166:0:2401:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941505.281168:0:2401:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941505.281169:0:2401:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941505.281173:0:2401:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c3b80. 00000020:00000040:1.0:1773941505.281175:0:2401:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941505.281177:0:2401:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941505.281178:0:2401:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941505.281181:0:2401:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941505.281182:0:2401:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941505.281184:0:2401:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941505.281188:0:2401:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d1118a000. 00000100:00000001:1.0:1773941505.281191:0:2401:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941505.281196:0:2401:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c3b80 x1860112291478144/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941505.281210:0:2401:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941505.281212:0:2401:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941505.281215:0:2401:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e15b8eeb] to pc [ptlrpcd_rcv+-1] req@ffff988d203c3b80 x1860112291478144/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941505.281230:0:2401:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941505.281232:0:2401:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941505.281233:0:2401:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:1.0:1773941505.281237:0:2401:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c2300 x1860112291478016/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941521 ref 1 fl Rpc:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941505.281243:0:2401:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941505.281245:0:2401:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941505.281247:0:2401:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941505.281250:0:2401:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c2300 x1860112291478016/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941521 ref 1 fl Interpret:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941505.281256:0:2401:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2401:1860112291478016:192.168.204.114@tcp:400:kworker.0 00000100:00000001:1.0:1773941505.281259:0:2401:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941505.281261:0:2401:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941505.281264:0:2401:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c2300 x1860112291478016/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941521 ref 1 fl Complete:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941505.281268:0:2401:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941505.281271:0:2401:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1773941505.281273:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:1.0:1773941505.281274:0:2401:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff988d181fc200. 02000000:00000001:1.0:1773941505.281278:0:2401:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1773941505.281280:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1773941505.281281:0:2401:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:3.0:1773941505.281284:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000020:00000040:1.0:1773941505.281285:0:2401:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941505.281289:0:2401:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941505.281292:0:2401:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff988d02cfd800. 00000100:00000040:3.0:1773941505.281296:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c3b80 x1860112291478144/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1773941505.281297:0:2401:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941505.281300:0:2401:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941505.281307:0:2401:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c2300. 00000100:00000001:1.0:1773941505.281313:0:2401:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941505.281315:0:2401:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941505.281317:0:2401:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941505.281320:0:2401:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941505.281321:0:2401:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941505.281325:0:2401:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941505.281328:0:2401:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941505.281330:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941505.281334:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941505.281338:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941505.281341:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941505.281344:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941505.281350:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291478144:192.168.204.114@tcp:250:ptlrpcd_00_00.0 00000100:00000001:3.0:1773941505.281355:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941505.281358:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941505.281361:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941505.281363:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941505.281368:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03523800. 02000000:00000001:3.0:1773941505.281370:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941505.281375:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941505.281380:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00000400:00000010:3.0:1773941505.281385:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00000100:00000200:3.0:1773941505.281389:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291478144, portal 25 00000100:00000001:3.0:1773941505.281391:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941505.281400:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631469952 : -113743078081664 : ffff988d203c3b80) 00000100:00000040:3.0:1773941505.281407:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c3b80 x1860112291478144/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941515 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941505.281416:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941505.281418:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:3.0:1773941505.281422:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182aca18. 00000100:00000200:3.0:1773941505.281426:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291478144, offset 0 00000400:00000200:3.0:1773941505.281432:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941505.281437:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941505.281443:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941505.281447:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941505.281451:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941505.281453:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941505.281456:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941505.281460:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[12, 13] 00000400:00000200:3.0:1773941505.281464:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941505.281467:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [124:124:256:4294967295] 192.168.204.114@tcp LPNI seq info [14:14:8:4294967295] 00000400:00000200:3.0:1773941505.281472:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941505.281480:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:3.0:1773941505.281486:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:3.0:1773941505.281491:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74c00. 00000800:00000200:3.0:1773941505.281497:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:3.0:1773941505.281504:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:3.0:1773941505.281508:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74c00 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:3.0:1773941505.281523:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941505.281527:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:3.0:1773941505.281529:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294896877 00000100:00000001:3.0:1773941505.281536:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294896871 00000100:00000001:3.0:1773941505.281541:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294896877 00000100:00000001:3.0:1773941505.281545:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294896871 00000100:00000001:3.0:1773941505.281557:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941505.281559:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941505.281562:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941505.281565:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941505.281570:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941505.281572:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1773941505.281575:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941505.281577:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941505.281579:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941505.281581:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941505.281597:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941505.281602:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74c00. 00000400:00000200:0.0:1773941505.281607:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941505.281612:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941505.281615:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182aca18 00000400:00000010:0.0:1773941505.281618:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182aca18. 00000100:00000001:0.0:1773941505.281621:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941505.281626:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c3b80 x1860112291478144/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941515 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941505.281636:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941505.281639:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c3b80 x1860112291478144/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941515 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941505.281647:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941505.281649:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941505.286992:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941505.287005:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941505.287014:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941505.287016:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941505.287022:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941505.287028:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941505.287033:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941505.287037:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941505.287041:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941505.287044:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324603a80 00000400:00000200:0.0:1773941505.287047:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941505.287051:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xb49 [1] + 0 00000800:00000001:0.0:1773941505.287057:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941505.287063:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941505.287065:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941505.287069:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941505.287073:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941505.287080:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c3b80 x1860112291478144/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941515 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941505.287089:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c3b80 x1860112291478144/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941515 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941505.287115:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941505.287121:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941505.287124:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941505.287206:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941505.287210:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941505.287218:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941505.287220:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941505.287222:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941505.287227:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941505.287229:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941505.287232:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941505.287236:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c3b80 x1860112291478144/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941515 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941505.287245:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c3b80 x1860112291478144/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941515 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941505.287251:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941505.287252:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941505.287255:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941505.287258:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941505.287260:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941505.287262:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941505.287265:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941505.287270:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=6 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941505.287273:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941505.287275:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941505.287277:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941505.287279:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941505.287282:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941505.287284:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941505.287293:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c3b80 x1860112291478144/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941515 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941505.287298:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941505.287300:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941505.287301:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941505.287304:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941505.287310:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941505.287312:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941505.287313:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941505.287316:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941505.287318:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941505.287319:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941505.287322:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941505.287325:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941505 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941505.287327:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941505.287333:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941505.287334:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941505.287337:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941505.287339:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941505.287340:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941505.287342:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941505.287345:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941505.287347:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941505.287350:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941505.287353:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941505.287355:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941505.287585:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941505.287589:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941505.287593:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c3b80 x1860112291478144/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941515 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941505.287601:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291478144:192.168.204.114@tcp:250:ptlrpcd_00_00.0 00000100:00000001:3.0:1773941505.287610:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941505.287613:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1773941505.287615:0:14717:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000040:3.0:1773941505.287617:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c3b80 x1860112291478144/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941515 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941505.287624:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941505.287626:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00080000:1.0:1773941505.287630:0:14717:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 02000000:00000010:3.0:1773941505.287631:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03523800. 00000100:00100000:1.0:1773941505.287638:0:14717:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000001:3.0:1773941505.287639:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941505.287643:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:1.0:1773941505.287645:0:14717:0:(obd_class.h:1215:obd_import_event()) Process entered 00000020:00000040:3.0:1773941505.287646:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 10000000:01000000:1.0:1773941505.287649:0:14717:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000020:00000001:3.0:1773941505.287652:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:1.0:1773941505.287654:0:14717:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000010:3.0:1773941505.287658:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d1118a000. 00000001:00000001:1.0:1773941505.287659:0:14717:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:3.0:1773941505.287666:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000001:00000001:1.0:1773941505.287668:0:14717:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941505.287670:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000001:00000001:1.0:1773941505.287674:0:14717:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000010:3.0:1773941505.287675:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c3b80. 00000001:00000001:1.0:1773941505.287678:0:14717:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:3.0:1773941505.287682:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000001:00000001:1.0:1773941505.287683:0:14717:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941505.287686:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941505.287688:0:14717:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941505.287691:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941505.287693:0:14717:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941505.287701:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:1.0:1773941505.287701:0:14717:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:3.0:1773941505.287706:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941505.287707:0:14717:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:3.0:1773941505.287710:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941505.287713:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941505.287713:0:14717:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941505.287719:0:14717:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941505.287724:0:14717:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941505.287728:0:14717:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941505.287729:0:14717:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941505.287731:0:14717:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:1.0:1773941505.287757:0:14717:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941505.287759:0:14717:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941505.287761:0:14717:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941505.287765:0:14717:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:1.0:1773941505.291360:0:14717:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941505.291365:0:14717:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941505.291367:0:14717:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941505.291371:0:14717:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941505.291381:0:14717:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941505.747122:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:2.0:1773941505.747132:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941505.747137:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941505.747140:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941505.747141:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941505.747143:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:2.0:1773941505.747146:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:2.0:1773941505.747148:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:2.0:1773941505.747151:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:2.0:1773941505.747154:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:2.0:1773941505.747157:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:2.0:1773941505.747168:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06507480. 00000020:00000040:2.0:1773941505.747189:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941505.747195:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941505.747198:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941505.747203:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941505.747213:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d08a4fe00. 00000100:00000001:2.0:1773941505.747220:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941505.747226:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:2.0:1773941505.747230:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:2.0:1773941505.747238:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d071ee100. 00010000:00000001:2.0:1773941505.747244:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:2.0:1773941505.747247:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d089dfa00. 00000020:00000001:2.0:1773941505.747252:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:2.0:1773941505.747255:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d089dfa00 with handle 0x8e86767622e28f7 to hash 00000020:00000001:2.0:1773941505.747257:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:2.0:1773941505.747259:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941505.747263:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941505.747267:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941505.747271:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941505.747276:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28f7 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941505.747287:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28f7 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941505.747310:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28f7 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941505.747322:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:2.0:1773941505.747324:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:2.0:1773941505.747331:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d09350540. 00000100:00000001:2.0:1773941505.747333:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330245121344 : -113743464430272 : ffff988d09350540) 00000100:00000001:2.0:1773941505.747337:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941505.747338:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196587648 : -113743512963968 : ffff988d06507480) 00000100:00000001:2.0:1773941505.747343:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941505.747353:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941505.747357:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:2.0:1773941505.747359:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941505.747369:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06507480 x1860112291478272/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941505.747380:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941505.747383:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941505.747387:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941505.747390:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941505.747393:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941505.747397:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06507480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291478272:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941505.747402:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941505.747408:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941505.747411:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941505.747414:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941505.747418:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a7800. 02000000:00000001:2.0:1773941505.747421:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941505.747428:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941505.747433:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000400:00000010:2.0:1773941505.747441:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000200:2.0:1773941505.747448:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291478272, portal 25 00000100:00000001:2.0:1773941505.747450:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941505.747452:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196587648 : -113743512963968 : ffff988d06507480) 00000100:00000040:2.0:1773941505.747458:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06507480 x1860112291478272/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941521 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941505.747467:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941505.747471:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:2.0:1773941505.747474:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516f68. 00000100:00000200:2.0:1773941505.747478:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291478272, offset 0 00000400:00000200:2.0:1773941505.747485:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941505.747494:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941505.747502:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941505.747508:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941505.747519:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941505.747523:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941505.747526:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941505.747530:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[14, 13] 00000400:00000200:2.0:1773941505.747534:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:2.0:1773941505.747540:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [125:125:256:4294967295] 192.168.204.124@tcp LPNI seq info [15:15:8:4294967295] 00000400:00000200:2.0:1773941505.747545:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:2.0:1773941505.747557:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:2.0:1773941505.747563:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:2.0:1773941505.747569:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0db000. 00000800:00000200:2.0:1773941505.747577:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:2.0:1773941505.747584:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:2.0:1773941505.747589:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0db000 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:2.0:1773941505.747607:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941505.747612:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941505.747614:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294897343 00000100:00000001:2.0:1773941505.747622:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294896877 00000100:00000001:2.0:1773941505.747629:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294897343 00000100:00000001:2.0:1773941505.747633:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294896877 00000100:00000001:2.0:1773941505.747642:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941505.747646:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941505.747649:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941505.747651:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:2.0:1773941505.747655:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d09350540 going to sleep for 11 seconds 00000100:00000001:2.0:1773941505.747659:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941505.747663:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941505.747751:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941505.747759:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0db000. 00000400:00000200:0.0:1773941505.747766:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941505.747774:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941505.747780:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516f68 00000400:00000010:0.0:1773941505.747783:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516f68. 00000100:00000001:0.0:1773941505.747787:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941505.747794:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06507480 x1860112291478272/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941521 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941505.747804:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941505.747819:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d06507480 x1860112291478272/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941521 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941505.747824:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941505.747826:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941505.748888:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941505.748899:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941505.748902:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941505.748903:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941505.748910:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941505.748914:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941505.748919:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941505.748923:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941505.748928:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941505.748931:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324603b00 00000400:00000200:0.0:1773941505.748935:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941505.748939:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xb51 [1] + 224 00000800:00000001:0.0:1773941505.748947:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941505.748955:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941505.748957:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941505.749005:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941505.749009:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941505.749012:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06507480 x1860112291478272/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941521 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941505.749020:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d06507480 x1860112291478272/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941521 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941505.749037:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941505.749042:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941505.749045:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941505.749114:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941505.749120:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941505.749121:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941505.749124:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941505.749127:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941505.749134:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941505.749137:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941505.749142:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06507480 x1860112291478272/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941521 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941505.749149:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06507480 x1860112291478272/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941521 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941505.749155:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941505.749156:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941505.749158:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941505.749160:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941505.749162:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941505.749163:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941505.749165:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941505.749170:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=6 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941505.749173:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=0 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941505.749175:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:2.0:1773941505.749177:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:2.0:1773941505.753012:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941505.753017:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:2.0:1773941505.753020:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1773941505.753024:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1773941505.753027:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1773941505.753030:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941505.753032:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941505.753033:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941505.753034:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:2.0:1773941505.753038:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d06507480 x1860112291478272/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941521 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941505.753044:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941505.753045:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941505.753046:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941505.753049:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941505.753059:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:2.0:1773941505.753060:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941505.753062:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941505.753063:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941505.753064:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941505.753065:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941505.753066:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941505.753068:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1773941505.753069:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1773941505.753072:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941505.753075:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 228 00000400:00000200:2.0:1773941505.753080:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941505.753099:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:2.0:1773941505.753103:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941505.753106:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941505.753108:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941505.753110:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:2.0:1773941505.753113:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941505.753114:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941505.753116:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941505.753118:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941505.753120:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941505.753121:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:2.0:1773941505.753123:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941505.753124:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941505.753125:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941505.753126:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:2.0:1773941505.753128:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:2.0:1773941505.753131:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:2.0:1773941505.753132:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941505.753134:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941505.753137:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941505.753139:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941505.753144:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941505.753146:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941505.753150:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06506a00. 00000020:00000040:2.0:1773941505.753153:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:2.0:1773941505.753156:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941505.753157:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941505.753159:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941505.753161:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941505.753163:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941505.753167:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d035a6400. 00000100:00000001:2.0:1773941505.753170:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941505.753174:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d06506a00 x1860112291478400/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941505.753181:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941505.753184:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941505.753187:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6b603d8] to pc [ptlrpcd_rcv+-1] req@ffff988d06506a00 x1860112291478400/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941505.753201:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941505.753203:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:2.0:1773941505.753204:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941505.753207:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941505.753208:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941505.753210:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941505.753211:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d09350540 going to sleep for 1 seconds 00000100:00000001:2.0:1773941505.753213:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941505.753215:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941505.753218:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06507480 x1860112291478272/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941521 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941505.753224:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941505.753226:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941505.753229:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941505.753231:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941505.753234:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06507480 x1860112291478272/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941521 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941505.753239:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941505.753244:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941505.753246:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941505.753255:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06506a00 x1860112291478400/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941505.753255:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06507480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291478272:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941505.753262:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941505.753267:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941505.753269:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941505.753270:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:3.0:1773941505.753272:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941505.753274:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1773941505.753276:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941505.753280:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1773941505.753280:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06507480 x1860112291478272/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941521 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1773941505.753287:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941505.753294:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291478400:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941505.753295:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941505.753300:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000010:2.0:1773941505.753301:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d09350540. 00000100:00000001:2.0:1773941505.753304:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:2.0:1773941505.753305:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:2.0:1773941505.753307:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:2.0:1773941505.753311:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 02000000:00000001:3.0:1773941505.753312:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:2.0:1773941505.753313:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:3.0:1773941505.753315:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941505.753316:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089dfa00 refcount=4 02000000:00000001:3.0:1773941505.753320:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:2.0:1773941505.753321:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941505.753325:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00010000:2.0:1773941505.753328:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28f7 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000010:3.0:1773941505.753335:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03523800. 00010000:00000001:2.0:1773941505.753335:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 02000000:00000001:3.0:1773941505.753339:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941505.753342:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28f7 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941505.753347:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941505.753352:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00010000:00000001:2.0:1773941505.753353:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:2.0:1773941505.753357:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28f7 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:3.0:1773941505.753359:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00000100:00000200:3.0:1773941505.753365:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291478400, portal 25 00010000:00000001:2.0:1773941505.753365:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941505.753368:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941505.753369:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584960 : -113743512966656 : ffff988d06506a00) 00000100:00000040:3.0:1773941505.753375:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06506a00 x1860112291478400/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941515 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1773941505.753375:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:2.0:1773941505.753379:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28f7 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941505.753394:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:2.0:1773941505.753397:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000040:3.0:1773941505.753398:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00000001:2.0:1773941505.753401:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1773941505.753402:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac880. 00010000:00000001:2.0:1773941505.753404:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000100:00000200:3.0:1773941505.753406:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291478400, offset 0 00010000:00010000:2.0:1773941505.753408:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28f7 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941505.753415:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00010000:2.0:1773941505.753417:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d089dfa00 already unused, calling callback (ffffffffc12b4c20) 10000000:00000001:2.0:1773941505.753421:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:3.0:1773941505.753422:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 10000000:00010000:2.0:1773941505.753426:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28f7 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941505.753429:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941505.753437:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:2.0:1773941505.753437:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:2.0:1773941505.753440:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:2.0:1773941505.753442:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941505.753443:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089dfa00 refcount=5 00000020:00000001:2.0:1773941505.753447:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00000400:00000200:3.0:1773941505.753449:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00000001:2.0:1773941505.753450:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941505.753454:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:3.0:1773941505.753455:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00010000:2.0:1773941505.753458:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28f7 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941505.753459:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941505.753467:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[14, 15] 10000000:00000001:2.0:1773941505.753471:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:3.0:1773941505.753473:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 10000000:00010000:2.0:1773941505.753475:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28f7 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941505.753480:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [126:126:256:4294967295] 192.168.204.114@tcp LPNI seq info [16:16:8:4294967295] 10000000:01000000:2.0:1773941505.753487:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:3.0:1773941505.753490:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 10000000:00000040:2.0:1773941505.753492:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:2.0:1773941505.753495:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941505.753499:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:2.0:1773941505.753502:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:3.0:1773941505.753503:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:3.0:1773941505.753509:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00010000:00000001:2.0:1773941505.753510:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1773941505.753513:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000800:00000010:3.0:1773941505.753514:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74c00. 00010000:00000001:2.0:1773941505.753515:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941505.753518:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941505.753521:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d089dfa00 with handle 0x8e86767622e28f7 from hash 00000800:00000200:3.0:1773941505.753524:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00010000:00000001:2.0:1773941505.753525:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1773941505.753527:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941505.753528:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941505.753529:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1773941505.753530:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:2.0:1773941505.753532:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:2.0:1773941505.753535:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941505.753536:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941505.753537:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941505.753541:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 10000000:00000001:2.0:1773941505.753541:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941505.753547:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74c00 type 1, nob 616 niov 1 nkiov 1 00010000:00010000:2.0:1773941505.753547:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28f7 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941505.753556:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941505.753557:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941505.753558:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:2.0:1773941505.753559:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:2.0:1773941505.753560:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941505.753560:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941505.753561:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1773941505.753563:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28f7 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941505.753564:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941505.753573:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:3.0:1773941505.753577:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294897349 00010000:00000040:2.0:1773941505.753577:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d071ee100 count: 0 00000100:00000001:3.0:1773941505.753586:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294897343 00000100:00000001:3.0:1773941505.753592:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294897349 00010000:00000001:2.0:1773941505.753599:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:3.0:1773941505.753600:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294897343 00010000:00000001:2.0:1773941505.753603:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941505.753605:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941505.753608:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1773941505.753610:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941505.753614:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941505.753618:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941505.753618:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06507480 x1860112291478272/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941521 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941505.753627:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941505.753627:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1773941505.753632:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 02000000:00000001:2.0:1773941505.753633:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1773941505.753635:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941505.753637:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941505.753638:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941505.753642:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941505.753643:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a7800. 02000000:00000001:2.0:1773941505.753647:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941505.753649:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941505.753650:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941505.753652:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941505.753655:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d08a4fe00. 02000000:00000001:2.0:1773941505.753659:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941505.753660:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941505.753662:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06507480. 00000100:00000001:2.0:1773941505.753664:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941505.753666:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:2.0:1773941505.753668:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:2.0:1773941505.753671:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:2.0:1773941505.753675:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000800:00000200:0.0:1773941505.753675:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 10000000:00000010:2.0:1773941505.753684:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d05437dc0. 00000800:00000010:0.0:1773941505.753684:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74c00. 00000020:00000010:2.0:1773941505.753694:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d08a4fe00. 00000400:00000200:0.0:1773941505.753694:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000010:2.0:1773941505.753705:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d08a4f400. 00000400:00000200:0.0:1773941505.753710:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000040:00000010:2.0:1773941505.753714:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d03356200. 00000020:00000010:2.0:1773941505.753721:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d09305000. 00000400:00000200:0.0:1773941505.753725:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac880 00000020:00000010:2.0:1773941505.753729:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d0381e000. 00000400:00000010:0.0:1773941505.753731:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac880. 00000100:00000001:0.0:1773941505.753739:0:2393:0:(events.c:34:request_out_callback()) Process entered 80000000:00000010:2.0:1773941505.753740:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d08bbb420. 00040000:00000010:2.0:1773941505.753745:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d0381f200. 00000100:00000200:0.0:1773941505.753749:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06506a00 x1860112291478400/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941515 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 20000000:00000010:2.0:1773941505.753750:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d0381e800. 10000000:00000040:2.0:1773941505.753759:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:2.0:1773941505.753767:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:2.0:1773941505.753779:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d08a4f400. 00000100:00000001:0.0:1773941505.753781:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000040:00000010:2.0:1773941505.753785:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d03356200. 00000020:00000010:2.0:1773941505.753793:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d09305000. 00000100:00000040:0.0:1773941505.753794:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06506a00 x1860112291478400/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941515 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1773941505.753800:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d0381e000. 80000000:00000010:2.0:1773941505.753808:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d08bbb420. 00040000:00000010:2.0:1773941505.753816:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d0381f200. 00000100:00000001:0.0:1773941505.753819:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 20000000:00000010:2.0:1773941505.753820:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d0381e800. 00000020:00000010:2.0:1773941505.753827:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d08a4fe00. 00000100:00000001:0.0:1773941505.753828:0:2393:0:(events.c:68:request_out_callback()) Process leaving 10000000:00000010:2.0:1773941505.753836:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d05437dc0. 10000000:01000000:2.0:1773941505.753843:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:2.0:1773941505.753849:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:2.0:1773941505.753852:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:2.0:1773941505.753854:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:2.0:1773941505.753856:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:2.0:1773941505.753857:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941505.753859:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941505.755022:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941505.755029:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941505.755031:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941505.755033:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941505.755037:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941505.755040:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941505.755045:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941505.755048:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941505.755051:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941505.755053:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324603b80 00000400:00000200:0.0:1773941505.755056:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941505.755059:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xb59 [1] + 0 00000800:00000001:0.0:1773941505.755064:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941505.755069:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941505.755071:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941505.755074:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941505.755077:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941505.755096:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06506a00 x1860112291478400/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941515 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941505.755106:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d06506a00 x1860112291478400/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941515 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941505.755134:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941505.755149:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941505.755155:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941505.755163:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941505.755166:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941505.755169:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941505.755171:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941505.755174:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941505.755178:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941505.755180:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941505.755182:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941505.755187:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06506a00 x1860112291478400/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941515 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941505.755195:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06506a00 x1860112291478400/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941515 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941505.755200:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941505.755201:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941505.755203:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941505.755210:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941505.755212:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941505.755213:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941505.755215:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941505.755219:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=6 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941505.755221:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=0 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941505.755223:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941505.755224:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941505.755226:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941505.755227:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941505.755229:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941505.755233:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06506a00 x1860112291478400/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941515 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941505.755237:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941505.755239:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941505.755240:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941505.755242:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941505.755246:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941505.755247:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941505.755249:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941505.755250:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941505.755251:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941505.755253:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941505.755255:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941505.755257:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941505 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941505.755259:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941505.755264:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941505.755265:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941505.755267:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941505.755268:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941505.755269:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941505.755271:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941505.755273:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941505.755275:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941505.755281:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941505.755283:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941505.755285:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941505.755516:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941505.755522:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941505.755527:0:14718:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000040:2.0:1773941505.755532:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06506a00 x1860112291478400/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941515 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1773941505.755536:0:14718:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:3.0:1773941505.755541:0:14718:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00100000:2.0:1773941505.755551:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291478400:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941505.755563:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941505.755566:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1773941505.755571:0:14718:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000040:2.0:1773941505.755572:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06506a00 x1860112291478400/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941515 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:01000000:3.0:1773941505.755574:0:14718:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:3.0:1773941505.755578:0:14718:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941505.755582:0:14718:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941505.755585:0:14718:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941505.755587:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:3.0:1773941505.755588:0:14718:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941505.755592:0:14718:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941505.755593:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:3.0:1773941505.755594:0:14718:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941505.755597:0:14718:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941505.755598:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03523800. 00000100:00000001:3.0:1773941505.755599:0:14718:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941505.755604:0:14718:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 02000000:00000001:2.0:1773941505.755605:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1773941505.755607:0:14718:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000020:00000001:2.0:1773941505.755609:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:3.0:1773941505.755610:0:14718:0:(recover.c:147:ptlrpc_resend()) Process entered 00000020:00000040:2.0:1773941505.755613:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941505.755614:0:14718:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1773941505.755616:0:14718:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:3.0:1773941505.755622:0:14718:0:(obd_class.h:1215:obd_import_event()) Process entered 00000020:00000001:2.0:1773941505.755622:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 10000000:01000000:3.0:1773941505.755626:0:14718:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:3.0:1773941505.755628:0:14718:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 02000000:00000010:2.0:1773941505.755628:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d035a6400. 02000000:00000001:2.0:1773941505.755634:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941505.755635:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941505.755637:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06506a00. 00000100:00000001:2.0:1773941505.755641:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941505.755642:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1773941505.755644:0:14718:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:00000001:2.0:1773941505.755645:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:3.0:1773941505.755646:0:14718:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941505.755650:0:14718:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941505.755652:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:02000000:3.0:1773941505.755654:0:14718:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:2.0:1773941505.755657:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941505.755660:0:14718:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941505.755663:0:14718:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941505.755665:0:14718:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941505.755665:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1773941505.755670:0:14718:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941505.755670:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941505.755673:0:14718:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:2.1:1773941505.758992:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d071ee100. 00010000:00000010:2.1:1773941505.758998:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d089dfa00. 00000020:00000001:2.0:1773941506.043147:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941506.043182:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941506.043186:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:2.0:1773941507.003174:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:2.0:1773941507.003180:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941507.003184:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941507.003186:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941507.003187:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941507.003188:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:2.0:1773941507.003191:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:2.0:1773941507.003193:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:2.0:1773941507.003195:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:2.0:1773941507.003197:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:2.0:1773941507.003198:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:2.0:1773941507.003209:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06506a00. 00000020:00000040:2.0:1773941507.003213:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941507.003217:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941507.003219:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941507.003224:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941507.003230:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d0381f200. 00000100:00000001:2.0:1773941507.003235:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941507.003239:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:2.0:1773941507.003242:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:2.0:1773941507.003247:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d071ee100. 00010000:00000001:2.0:1773941507.003252:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:2.0:1773941507.003254:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d089dfa00. 00000020:00000001:2.0:1773941507.003257:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:2.0:1773941507.003260:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d089dfa00 with handle 0x8e86767622e28fe to hash 00000020:00000001:2.0:1773941507.003261:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:2.0:1773941507.003262:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941507.003265:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941507.003268:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941507.003271:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941507.003274:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28fe lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941507.003281:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28fe lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941507.003317:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28fe lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941507.003322:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:2.0:1773941507.003324:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:2.0:1773941507.003329:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d09350540. 00000100:00000001:2.0:1773941507.003331:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330245121344 : -113743464430272 : ffff988d09350540) 00000100:00000001:2.0:1773941507.003333:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941507.003334:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584960 : -113743512966656 : ffff988d06506a00) 00000100:00000001:2.0:1773941507.003339:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941507.003342:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941507.003343:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:2.0:1773941507.003345:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941507.003353:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06506a00 x1860112291478528/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941507.003360:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941507.003362:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941507.003364:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941507.003366:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941507.003368:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941507.003371:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291478528:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941507.003374:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941507.003378:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941507.003379:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941507.003382:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941507.003385:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a7800. 02000000:00000001:2.0:1773941507.003387:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941507.003393:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941507.003396:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000400:00000010:2.0:1773941507.003401:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000200:2.0:1773941507.003406:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291478528, portal 25 00000100:00000001:2.0:1773941507.003407:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941507.003412:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584960 : -113743512966656 : ffff988d06506a00) 00000100:00000040:2.0:1773941507.003416:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06506a00 x1860112291478528/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941523 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941507.003421:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941507.003423:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:2.0:1773941507.003425:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045163b8. 00000100:00000200:2.0:1773941507.003428:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291478528, offset 0 00000400:00000200:2.0:1773941507.003433:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941507.003440:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941507.003446:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941507.003451:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941507.003454:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941507.003457:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941507.003459:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941507.003462:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[16, 15] 00000400:00000200:2.0:1773941507.003464:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:2.0:1773941507.003467:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [127:127:256:4294967295] 192.168.204.124@tcp LPNI seq info [17:17:8:4294967295] 00000400:00000200:2.0:1773941507.003470:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:2.0:1773941507.003478:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:2.0:1773941507.003483:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:2.0:1773941507.003487:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0db900. 00000800:00000200:2.0:1773941507.003493:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:2.0:1773941507.003499:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:2.0:1773941507.003502:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0db900 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:2.0:1773941507.003519:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941507.003523:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941507.003525:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294898599 00000100:00000001:2.0:1773941507.003532:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294897349 00000100:00000001:2.0:1773941507.003537:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294898599 00000100:00000001:2.0:1773941507.003540:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294897349 00000100:00000001:2.0:1773941507.003549:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941507.003551:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941507.003553:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941507.003555:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:2.0:1773941507.003557:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d09350540 going to sleep for 11 seconds 00000100:00000001:2.0:1773941507.003563:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941507.003566:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941507.003671:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941507.003682:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0db900. 00000400:00000200:0.0:1773941507.003691:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941507.003702:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941507.003710:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045163b8 00000400:00000010:0.0:1773941507.003713:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045163b8. 00000100:00000001:0.0:1773941507.003719:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941507.003727:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06506a00 x1860112291478528/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941523 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941507.003741:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941507.003746:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d06506a00 x1860112291478528/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941523 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941507.003754:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941507.003757:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941507.004759:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941507.004773:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941507.004776:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941507.004779:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941507.004786:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941507.004792:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941507.004800:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941507.004805:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941507.004810:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941507.004816:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324603c00 00000400:00000200:0.0:1773941507.004822:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941507.004830:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xb61 [1] + 224 00000800:00000001:0.0:1773941507.004839:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941507.004849:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941507.004852:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941507.004855:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941507.004861:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941507.004878:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06506a00 x1860112291478528/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941523 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941507.004889:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d06506a00 x1860112291478528/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941523 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941507.004910:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941507.004919:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941507.004924:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941507.004940:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941507.004952:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941507.004954:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941507.004958:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941507.005013:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941507.005015:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941507.005018:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941507.005024:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06506a00 x1860112291478528/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941523 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941507.005036:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06506a00 x1860112291478528/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941523 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941507.005043:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941507.005045:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941507.005048:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941507.005051:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941507.005054:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941507.005056:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941507.005070:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941507.005076:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=8 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941507.005079:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941507.005082:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:2.0:1773941507.005087:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:2.0:1773941507.005090:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941507.005093:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:2.0:1773941507.005096:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1773941507.005100:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1773941507.005105:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1773941507.005115:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941507.005118:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941507.005120:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941507.005122:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:2.0:1773941507.005126:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d06506a00 x1860112291478528/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941523 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941507.005132:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941507.005133:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941507.005135:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941507.005137:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941507.005139:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:2.0:1773941507.005140:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941507.005142:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941507.005142:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941507.005143:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941507.005144:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941507.005145:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941507.005147:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1773941507.005148:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1773941507.005150:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941507.005152:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 228 00000400:00000200:2.0:1773941507.005156:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941507.005159:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:2.0:1773941507.005162:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941507.005164:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941507.005166:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941507.005167:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:2.0:1773941507.005170:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941507.005171:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941507.005172:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941507.005174:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941507.005176:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941507.005178:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:2.0:1773941507.005183:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941507.005184:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941507.005186:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941507.005187:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:2.0:1773941507.005189:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:2.0:1773941507.005191:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:2.0:1773941507.005193:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941507.005195:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941507.005211:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941507.005213:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941507.005214:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941507.005216:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941507.005218:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06507480. 00000020:00000040:2.0:1773941507.005220:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:2.0:1773941507.005222:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941507.005224:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941507.005225:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941507.005227:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941507.005228:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941507.005232:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d035a6400. 00000100:00000001:2.0:1773941507.005235:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941507.005239:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d06507480 x1860112291478656/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941507.005244:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941507.005246:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941507.005249:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000006ea800dd] to pc [ptlrpcd_rcv+-1] req@ffff988d06507480 x1860112291478656/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941507.005267:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941507.005269:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:2.0:1773941507.005271:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941507.005273:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941507.005275:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941507.005282:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941507.005285:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d09350540 going to sleep for 1 seconds 00000100:00000001:2.0:1773941507.005287:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941507.005289:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941507.005295:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06506a00 x1860112291478528/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941523 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941507.005299:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941507.005302:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941507.005303:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941507.005305:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941507.005307:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941507.005310:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1773941507.005310:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941507.005314:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06506a00 x1860112291478528/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941523 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941507.005315:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06507480 x1860112291478656/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941507.005323:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:2.0:1773941507.005324:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291478528:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941507.005325:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941507.005328:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1773941507.005329:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1773941507.005330:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941507.005331:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:3.0:1773941507.005333:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941507.005334:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00100000:3.0:1773941507.005336:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06507480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291478656:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941507.005336:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1773941507.005339:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000040:2.0:1773941507.005339:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06506a00 x1860112291478528/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941523 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1773941507.005342:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941507.005343:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941507.005346:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1773941507.005347:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941507.005349:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03520000. 00000100:00000010:2.0:1773941507.005349:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d09350540. 02000000:00000001:3.0:1773941507.005351:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941507.005352:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:2.0:1773941507.005354:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000200:3.0:1773941507.005355:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:2.0:1773941507.005356:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000400:00000010:3.0:1773941507.005358:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00010000:00000001:2.0:1773941507.005359:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000010:3.0:1773941507.005361:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00000020:00000001:2.0:1773941507.005361:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941507.005363:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089dfa00 refcount=4 00000100:00000200:3.0:1773941507.005365:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291478656, portal 25 00000020:00000001:2.0:1773941507.005365:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00000100:00000001:3.0:1773941507.005367:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00000001:2.0:1773941507.005368:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00000100:00000001:3.0:1773941507.005369:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196587648 : -113743512963968 : ffff988d06507480) 00010000:00010000:2.0:1773941507.005373:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28fe lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:3.0:1773941507.005374:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06507480 x1860112291478656/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941517 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941507.005380:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941507.005383:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:3.0:1773941507.005386:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac440. 00010000:00000001:2.0:1773941507.005387:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000200:3.0:1773941507.005388:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291478656, offset 0 00010000:00010000:2.0:1773941507.005391:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28fe lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941507.005393:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941507.005398:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:2.0:1773941507.005398:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:2.0:1773941507.005401:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28fe lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941507.005402:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941507.005405:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941507.005408:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00000001:2.0:1773941507.005408:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:3.0:1773941507.005411:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:2.0:1773941507.005411:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:2.0:1773941507.005413:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28fe lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941507.005418:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:2.0:1773941507.005420:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:3.0:1773941507.005421:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[16, 17] 00010000:00000001:2.0:1773941507.005422:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941507.005425:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00010000:00000001:2.0:1773941507.005426:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:3.0:1773941507.005429:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [128:128:256:4294967295] 192.168.204.114@tcp LPNI seq info [18:18:8:4294967295] 00010000:00010000:2.0:1773941507.005429:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28fe lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941507.005434:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:2.0:1773941507.005436:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d089dfa00 already unused, calling callback (ffffffffc12b4c20) 10000000:00000001:2.0:1773941507.005439:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:2.0:1773941507.005442:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28fe lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941507.005443:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:3.0:1773941507.005448:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00010000:00000001:2.0:1773941507.005449:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:2.0:1773941507.005451:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000800:00000010:3.0:1773941507.005452:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74800. 00000020:00000001:2.0:1773941507.005452:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941507.005454:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089dfa00 refcount=5 00000020:00000001:2.0:1773941507.005456:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00000800:00000200:3.0:1773941507.005457:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00010000:00000001:2.0:1773941507.005458:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941507.005461:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000800:00000200:3.0:1773941507.005463:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:3.0:1773941507.005466:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74800 type 1, nob 616 niov 1 nkiov 1 00010000:00010000:2.0:1773941507.005468:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28fe lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:2.0:1773941507.005474:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:2.0:1773941507.005476:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28fe lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941507.005479:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941507.005482:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 10000000:01000000:2.0:1773941507.005483:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000100:00000001:3.0:1773941507.005485:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294898601 10000000:00000040:2.0:1773941507.005486:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:2.0:1773941507.005488:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941507.005489:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294898599 00010000:00010000:2.0:1773941507.005490:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000100:00000001:3.0:1773941507.005492:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294898601 00010000:00000001:2.0:1773941507.005493:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000100:00000001:3.0:1773941507.005496:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294898599 00010000:00000001:2.0:1773941507.005496:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1773941507.005498:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000100:00000001:3.0:1773941507.005499:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:2.0:1773941507.005500:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:3.0:1773941507.005501:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941507.005503:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941507.005504:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941507.005507:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1773941507.005507:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d089dfa00 with handle 0x8e86767622e28fe from hash 00010000:00000001:2.0:1773941507.005510:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1773941507.005511:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941507.005513:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:3.0:1773941507.005514:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:2.0:1773941507.005514:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000100:00000001:3.0:1773941507.005516:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00010000:00000001:2.0:1773941507.005518:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000100:00000001:3.0:1773941507.005520:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:2.0:1773941507.005520:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000100:00000001:3.0:1773941507.005521:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00010000:00000001:2.0:1773941507.005523:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941507.005524:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941507.005524:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:3.0:1773941507.005526:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941507.005526:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941507.005528:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941507.005531:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28fe lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941507.005539:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941507.005541:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941507.005542:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:2.0:1773941507.005544:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:2.0:1773941507.005545:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941507.005547:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941507.005554:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1773941507.005556:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e28fe lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:2.0:1773941507.005565:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d071ee100 count: 0 00000800:00000200:0.0:1773941507.005569:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941507.005575:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74800. 00010000:00000001:2.0:1773941507.005578:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:0.0:1773941507.005581:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000001:2.0:1773941507.005582:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941507.005588:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000400:00000200:0.0:1773941507.005592:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000040:2.0:1773941507.005594:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06506a00 x1860112291478528/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941523 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1773941507.005601:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac440 00000100:00000001:2.0:1773941507.005605:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000400:00000010:0.0:1773941507.005608:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac440. 02000000:00000001:2.0:1773941507.005610:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941507.005616:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a7800. 00000100:00000001:0.0:1773941507.005616:0:2393:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:2.0:1773941507.005623:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000200:0.0:1773941507.005628:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06507480 x1860112291478656/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941517 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1773941507.005629:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941507.005633:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941507.005644:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941507.005648:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d0381f200. 00000100:00000001:0.0:1773941507.005651:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1773941507.005655:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000040:0.0:1773941507.005662:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06507480 x1860112291478656/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941517 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1773941507.005663:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941507.005668:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06506a00. 00000100:00000001:2.0:1773941507.005679:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941507.005682:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:2.0:1773941507.005687:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941507.005687:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 10000000:01000000:2.0:1773941507.005696:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000100:00000001:0.0:1773941507.005697:0:2393:0:(events.c:68:request_out_callback()) Process leaving 10000000:00000001:2.0:1773941507.005700:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:2.0:1773941507.005704:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d05437dc0. 00000020:00000010:2.0:1773941507.005709:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d0381f200. 00000020:00000010:2.0:1773941507.005713:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d0381e000. 00000040:00000010:2.0:1773941507.005716:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d0381e800. 00000020:00000010:2.0:1773941507.005721:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941507.005726:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d03356200. 80000000:00000010:2.0:1773941507.005729:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d08bbb420. 00040000:00000010:2.0:1773941507.005733:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d08a4fe00. 20000000:00000010:2.0:1773941507.005736:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d08a4f400. 10000000:00000040:2.0:1773941507.005744:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:2.0:1773941507.005745:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:2.0:1773941507.005748:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d0381e000. 00000040:00000010:2.0:1773941507.005752:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d0381e800. 00000020:00000010:2.0:1773941507.005754:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941507.005757:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d03356200. 80000000:00000010:2.0:1773941507.005760:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d08bbb420. 00040000:00000010:2.0:1773941507.005762:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d08a4fe00. 20000000:00000010:2.0:1773941507.005765:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d08a4f400. 00000020:00000010:2.0:1773941507.005767:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0381f200. 10000000:00000010:2.0:1773941507.005769:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d05437dc0. 10000000:01000000:2.0:1773941507.005771:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:2.0:1773941507.005773:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:2.0:1773941507.005775:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 00000800:00000001:0.0:1773941507.006762:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941507.006768:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941507.006770:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941507.006772:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941507.006775:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941507.006779:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941507.006783:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941507.006786:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941507.006788:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941507.006791:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324603c80 00000400:00000200:0.0:1773941507.006795:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941507.006804:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xb69 [1] + 0 00000800:00000001:0.0:1773941507.006808:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941507.006813:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941507.006814:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941507.006817:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941507.006819:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941507.006822:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06507480 x1860112291478656/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941517 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941507.006830:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d06507480 x1860112291478656/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941517 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941507.006844:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941507.006848:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941507.006851:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941507.006883:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941507.006887:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941507.006891:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941507.006893:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941507.006896:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941507.006909:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941507.006911:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941507.006914:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941507.006919:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06507480 x1860112291478656/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941517 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941507.006926:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06507480 x1860112291478656/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941517 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941507.006931:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941507.006932:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941507.006934:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941507.006935:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941507.006936:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941507.006938:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941507.006940:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941507.006942:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=8 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941507.006944:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941507.006954:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941507.006955:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941507.006957:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941507.006958:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941507.006981:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941507.006985:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06507480 x1860112291478656/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941517 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941507.006990:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941507.006993:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941507.006994:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941507.006997:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941507.007001:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941507.007002:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941507.007003:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941507.007005:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941507.007006:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941507.007008:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941507.007009:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941507.007017:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941507 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941507.007019:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941507.007023:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941507.007025:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941507.007027:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941507.007028:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941507.007029:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941507.007032:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941507.007033:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941507.007035:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941507.007037:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941507.007040:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941507.007042:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941507.007290:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941507.007292:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941507.007296:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06507480 x1860112291478656/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941517 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941507.007302:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06507480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291478656:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941507.007306:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941507.007307:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941507.007310:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06507480 x1860112291478656/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941517 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941507.007315:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941507.007317:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941507.007319:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03520000. 02000000:00000001:3.0:1773941507.007324:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:1.0:1773941507.007324:0:14720:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000020:00000001:3.0:1773941507.007336:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941507.007337:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941507.007339:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941507.007340:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d035a6400. 02000000:00000001:3.0:1773941507.007344:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00080000:1.0:1773941507.007349:0:14720:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 02000000:00000001:3.0:1773941507.007354:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00100000:1.0:1773941507.007356:0:14720:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000010:3.0:1773941507.007358:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06507480. 00000100:00000001:1.0:1773941507.007369:0:14720:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:3.0:1773941507.007370:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941507.007377:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:01000000:1.0:1773941507.007378:0:14720:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000001:3.0:1773941507.007381:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941507.007388:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000001:00000001:1.0:1773941507.007388:0:14720:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:3.0:1773941507.007390:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941507.007395:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000001:00000001:1.0:1773941507.007395:0:14720:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:3.0:1773941507.007397:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941507.007401:0:14720:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941507.007404:0:14720:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941507.007406:0:14720:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941507.007410:0:14720:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941507.007412:0:14720:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941507.007414:0:14720:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:1.0:1773941507.007419:0:14720:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:1.0:1773941507.007422:0:14720:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941507.007424:0:14720:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941507.007426:0:14720:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941507.007428:0:14720:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941507.007430:0:14720:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941507.007431:0:14720:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941507.007432:0:14720:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:1.0:1773941507.007445:0:14720:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941507.007447:0:14720:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941507.007448:0:14720:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941507.007451:0:14720:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:1.0:1773941507.007453:0:14720:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941507.007456:0:14720:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941507.007457:0:14720:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941507.007459:0:14720:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941507.007460:0:14720:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941507.010341:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:2.0:1773941507.010345:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:2.0:1773941507.010348:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941507.010352:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00010000:00000010:2.1:1773941507.013016:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d071ee100. 00010000:00000010:2.1:1773941507.013025:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d089dfa00. 00000020:00000001:2.0:1773941507.067152:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941507.067158:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941507.067161:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1773941508.091167:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941508.091175:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941508.091180:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:2.0:1773941508.155102:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:2.0:1773941508.155110:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941508.155115:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941508.155117:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941508.155119:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941508.155120:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:2.0:1773941508.155123:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:2.0:1773941508.155125:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:2.0:1773941508.155127:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:2.0:1773941508.155129:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:2.0:1773941508.155131:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:2.0:1773941508.155142:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06506a00. 00000020:00000040:2.0:1773941508.155166:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941508.155171:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941508.155174:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941508.155180:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941508.155187:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d08a4f400. 00000100:00000001:2.0:1773941508.155195:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941508.155200:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:2.0:1773941508.155204:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:2.0:1773941508.155212:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d071ee100. 00010000:00000001:2.0:1773941508.155226:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:2.0:1773941508.155229:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d089dfa00. 00000020:00000001:2.0:1773941508.155233:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:2.0:1773941508.155236:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d089dfa00 with handle 0x8e86767622e2905 to hash 00000020:00000001:2.0:1773941508.155237:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:2.0:1773941508.155239:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941508.155241:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941508.155245:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941508.155248:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941508.155251:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2905 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941508.155260:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2905 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941508.155272:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2905 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941508.155280:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:2.0:1773941508.155282:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:2.0:1773941508.155290:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d09350540. 00000100:00000001:2.0:1773941508.155292:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330245121344 : -113743464430272 : ffff988d09350540) 00000100:00000001:2.0:1773941508.155296:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941508.155298:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584960 : -113743512966656 : ffff988d06506a00) 00000100:00000001:2.0:1773941508.155304:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941508.155315:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941508.155319:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:2.0:1773941508.155321:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941508.155335:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06506a00 x1860112291478784/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941508.155346:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941508.155349:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941508.155354:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941508.155357:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941508.155360:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941508.155365:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291478784:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941508.155369:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941508.155376:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941508.155379:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941508.155383:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941508.155389:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a4000. 02000000:00000001:2.0:1773941508.155391:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941508.155408:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941508.155414:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000400:00000010:2.0:1773941508.155423:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000200:2.0:1773941508.155431:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291478784, portal 25 00000100:00000001:2.0:1773941508.155433:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941508.155435:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584960 : -113743512966656 : ffff988d06506a00) 00000100:00000040:2.0:1773941508.155441:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06506a00 x1860112291478784/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941524 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941508.155449:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941508.155452:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:2.0:1773941508.155454:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516b28. 00000100:00000200:2.0:1773941508.155458:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291478784, offset 0 00000400:00000200:2.0:1773941508.155466:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941508.155476:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941508.155485:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941508.155492:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941508.155503:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941508.155513:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941508.155516:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941508.155520:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[18, 17] 00000400:00000200:2.0:1773941508.155524:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:2.0:1773941508.155529:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [129:129:256:4294967295] 192.168.204.124@tcp LPNI seq info [19:19:8:4294967295] 00000400:00000200:2.0:1773941508.155535:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:2.0:1773941508.155550:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:2.0:1773941508.155557:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:2.0:1773941508.155564:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0db600. 00000800:00000200:2.0:1773941508.155573:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:2.0:1773941508.155580:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:2.0:1773941508.155586:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0db600 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:2.0:1773941508.155610:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941508.155615:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941508.155618:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294899751 00000100:00000001:2.0:1773941508.155629:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294898601 00000100:00000001:2.0:1773941508.155636:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294899751 00000100:00000001:2.0:1773941508.155640:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294898601 00000100:00000001:2.0:1773941508.155645:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941508.155648:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941508.155651:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941508.155653:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:2.0:1773941508.155656:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d09350540 going to sleep for 11 seconds 00000100:00000001:2.0:1773941508.155660:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941508.155664:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941508.155722:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941508.155729:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0db600. 00000400:00000200:0.0:1773941508.155737:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941508.155745:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941508.155752:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516b28 00000400:00000010:0.0:1773941508.155755:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516b28. 00000100:00000001:0.0:1773941508.155759:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941508.155765:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06506a00 x1860112291478784/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941524 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941508.155777:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941508.155782:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d06506a00 x1860112291478784/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941524 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941508.155788:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941508.155790:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941508.156714:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941508.156726:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941508.156729:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941508.156732:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941508.156738:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941508.156744:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941508.156762:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941508.156767:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941508.156773:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941508.156778:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324603d00 00000400:00000200:0.0:1773941508.156783:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941508.156790:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xb71 [1] + 224 00000800:00000001:0.0:1773941508.156799:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941508.156810:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941508.156813:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941508.156818:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941508.156824:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941508.156829:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06506a00 x1860112291478784/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941524 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941508.156839:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d06506a00 x1860112291478784/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941524 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941508.156853:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941508.156860:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941508.156863:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941508.156887:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941508.156893:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941508.156895:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941508.156897:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941508.156900:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941508.156907:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941508.156910:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941508.156913:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06506a00 x1860112291478784/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941524 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941508.156921:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06506a00 x1860112291478784/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941524 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941508.156926:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941508.156927:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941508.156930:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941508.156932:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941508.156934:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941508.156936:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941508.156938:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941508.156943:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=9 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941508.156946:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941508.156948:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:2.0:1773941508.156951:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:2.0:1773941508.156954:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941508.156956:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:2.0:1773941508.156959:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1773941508.156979:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1773941508.156983:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1773941508.156986:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941508.156988:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941508.156989:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941508.156991:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:2.0:1773941508.156994:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d06506a00 x1860112291478784/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941524 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941508.156999:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941508.157000:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941508.157001:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941508.157004:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941508.157010:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:2.0:1773941508.157011:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941508.157013:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941508.157014:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941508.157015:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941508.157016:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941508.157017:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941508.157019:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1773941508.157021:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1773941508.157023:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941508.157025:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 230 00000400:00000200:2.0:1773941508.157028:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941508.157031:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:2.0:1773941508.157035:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941508.157048:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941508.157050:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941508.157052:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:2.0:1773941508.157055:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941508.157057:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941508.157058:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941508.157061:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941508.157062:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941508.157065:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:2.0:1773941508.157067:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941508.157068:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941508.157070:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941508.157071:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:2.0:1773941508.157073:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:2.0:1773941508.157075:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:2.0:1773941508.157077:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941508.157079:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941508.157082:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941508.157083:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941508.157089:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941508.157090:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941508.157094:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06506300. 00000020:00000040:2.0:1773941508.157096:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:2.0:1773941508.157099:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941508.157100:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941508.157102:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941508.157103:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941508.157105:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941508.157109:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d035a5c00. 00000100:00000001:2.0:1773941508.157111:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941508.157116:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d06506300 x1860112291478912/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941508.157122:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941508.157124:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941508.157127:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005464cf12] to pc [ptlrpcd_rcv+-1] req@ffff988d06506300 x1860112291478912/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941508.157143:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941508.157146:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:2.0:1773941508.157147:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941508.157149:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941508.157151:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941508.157153:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941508.157155:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d09350540 going to sleep for 1 seconds 00000100:00000001:2.0:1773941508.157157:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941508.157159:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941508.157163:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06506a00 x1860112291478784/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941524 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941508.157168:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941508.157170:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941508.157171:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941508.157175:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941508.157176:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941508.157179:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941508.157180:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941508.157190:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06506300 x1860112291478912/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941508.157190:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06506a00 x1860112291478784/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941524 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941508.157210:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941508.157213:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941508.157217:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291478784:192.168.204.114@tcp:101:ll_cfg_requeue.0 02000000:00000001:3.0:1773941508.157218:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941508.157221:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941508.157225:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941508.157225:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1773941508.157230:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06506300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291478912:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941508.157230:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941508.157233:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:3.0:1773941508.157236:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1773941508.157237:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1773941508.157241:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:2.0:1773941508.157244:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06506a00 x1860112291478784/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941524 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1773941508.157245:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941508.157249:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941508.157253:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03520000. 00000100:00000001:2.0:1773941508.157254:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941508.157258:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941508.157259:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d09350540. 00000100:00000001:2.0:1773941508.157263:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000400:00000200:3.0:1773941508.157264:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941508.157266:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000010:3.0:1773941508.157269:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00010000:00000001:2.0:1773941508.157272:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000400:00000010:3.0:1773941508.157274:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00010000:00000001:2.0:1773941508.157277:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000200:3.0:1773941508.157279:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291478912, portal 25 00000020:00000001:2.0:1773941508.157281:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:3.0:1773941508.157285:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000020:00000040:2.0:1773941508.157285:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089dfa00 refcount=4 00000100:00000001:3.0:1773941508.157289:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196583168 : -113743512968448 : ffff988d06506300) 00000020:00000001:2.0:1773941508.157291:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941508.157297:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00000100:00000040:3.0:1773941508.157299:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06506300 x1860112291478912/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941518 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:2.0:1773941508.157303:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2905 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941508.157317:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:2.0:1773941508.157321:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00010000:00010000:2.0:1773941508.157325:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2905 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:3.0:1773941508.157332:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:3.0:1773941508.157337:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac4c8. 00010000:00000001:2.0:1773941508.157339:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000200:3.0:1773941508.157343:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291478912, offset 0 00010000:00010000:2.0:1773941508.157344:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2905 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941508.157354:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00000001:2.0:1773941508.157358:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:3.0:1773941508.157362:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941508.157368:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:2.0:1773941508.157369:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:3.0:1773941508.157374:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00010000:2.0:1773941508.157375:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2905 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941508.157380:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941508.157387:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:2.0:1773941508.157389:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941508.157392:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941508.157393:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:2.0:1773941508.157397:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:3.0:1773941508.157401:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[18, 19] 00010000:00010000:2.0:1773941508.157401:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2905 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941508.157415:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d089dfa00 already unused, calling callback (ffffffffc12b4c20) 00000400:00000200:3.0:1773941508.157416:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 10000000:00000001:2.0:1773941508.157420:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:2.0:1773941508.157424:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2905 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941508.157425:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [130:130:256:4294967295] 192.168.204.114@tcp LPNI seq info [20:20:8:4294967295] 00010000:00000001:2.0:1773941508.157434:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:2.0:1773941508.157436:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:3.0:1773941508.157438:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000020:00000001:2.0:1773941508.157438:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941508.157441:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089dfa00 refcount=5 00000020:00000001:2.0:1773941508.157444:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941508.157449:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941508.157454:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:3.0:1773941508.157455:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00010000:00010000:2.0:1773941508.157457:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2905 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:2.0:1773941508.157465:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000800:00000200:3.0:1773941508.157466:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 10000000:00010000:2.0:1773941508.157468:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2905 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1773941508.157475:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000800:00000010:3.0:1773941508.157476:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74f00. 10000000:00000040:2.0:1773941508.157477:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:2.0:1773941508.157480:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941508.157483:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000800:00000200:3.0:1773941508.157485:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00010000:00000001:2.0:1773941508.157486:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000800:00000200:3.0:1773941508.157493:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00010000:00000001:2.0:1773941508.157494:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000800:00000200:3.0:1773941508.157498:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74f00 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:2.0:1773941508.157498:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1773941508.157502:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941508.157504:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941508.157506:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d089dfa00 with handle 0x8e86767622e2905 from hash 00010000:00000001:2.0:1773941508.157509:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1773941508.157510:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941508.157511:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941508.157511:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941508.157512:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1773941508.157515:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:2.0:1773941508.157517:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000400:00000200:3.0:1773941508.157518:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:2.0:1773941508.157521:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941508.157523:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294899753 00010000:00000001:2.0:1773941508.157524:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941508.157527:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941508.157530:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941508.157532:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294899751 00010000:00010000:2.0:1773941508.157532:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2905 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941508.157541:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294899753 00000100:00000001:3.0:1773941508.157546:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294899751 00010000:00000001:2.0:1773941508.157547:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941508.157550:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941508.157553:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000100:00000001:3.0:1773941508.157554:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:2.0:1773941508.157555:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:2.0:1773941508.157558:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941508.157561:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941508.157561:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941508.157562:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1773941508.157565:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2905 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941508.157567:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941508.157574:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000040:2.0:1773941508.157576:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d071ee100 count: 0 00000800:00000200:0.0:1773941508.157585:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00000001:2.0:1773941508.157588:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941508.157591:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000800:00000010:0.0:1773941508.157591:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74f00. 00000100:00000001:3.0:1773941508.157592:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941508.157593:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1773941508.157597:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000400:00000200:0.0:1773941508.157600:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1773941508.157603:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1773941508.157604:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06506a00 x1860112291478784/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941524 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941508.157608:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000200:0.0:1773941508.157611:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:3.0:1773941508.157612:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941508.157617:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941508.157621:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac4c8 00000100:00000001:2.0:1773941508.157626:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000400:00000010:0.0:1773941508.157627:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac4c8. 02000000:00000001:2.0:1773941508.157631:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1773941508.157633:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941508.157638:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06506300 x1860112291478912/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941518 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000010:2.0:1773941508.157643:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a4000. 00000100:00000001:0.0:1773941508.157647:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1773941508.157650:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000040:0.0:1773941508.157651:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06506300 x1860112291478912/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941518 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1773941508.157654:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941508.157659:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941508.157662:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941508.157663:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941508.157666:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d08a4f400. 02000000:00000001:2.0:1773941508.157670:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941508.157673:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1773941508.157674:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000010:2.0:1773941508.157676:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06506a00. 00000100:00000001:2.0:1773941508.157679:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941508.157681:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:2.0:1773941508.157683:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:2.0:1773941508.157685:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:2.0:1773941508.157687:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:2.0:1773941508.157691:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d05437dc0. 00000020:00000010:2.0:1773941508.157695:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d08a4f400. 00000020:00000010:2.0:1773941508.157699:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d08a4fe00. 00000040:00000010:2.0:1773941508.157705:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d03356200. 00000020:00000010:2.0:1773941508.157709:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941508.157713:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d0381f200. 80000000:00000010:2.0:1773941508.157717:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d08bbb420. 00040000:00000010:2.0:1773941508.157721:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d0381e800. 20000000:00000010:2.0:1773941508.157725:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d0381e000. 10000000:00000040:2.0:1773941508.157729:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:2.0:1773941508.157731:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:2.0:1773941508.157735:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d08a4fe00. 00000040:00000010:2.0:1773941508.157744:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d03356200. 00000020:00000010:2.0:1773941508.157747:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941508.157750:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d0381f200. 80000000:00000010:2.0:1773941508.157753:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d08bbb420. 00040000:00000010:2.0:1773941508.157755:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d0381e800. 20000000:00000010:2.0:1773941508.157758:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d0381e000. 00000020:00000010:2.0:1773941508.157760:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d08a4f400. 10000000:00000010:2.0:1773941508.157762:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d05437dc0. 10000000:01000000:2.0:1773941508.157764:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:2.0:1773941508.157772:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:2.0:1773941508.157773:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:2.0:1773941508.157775:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:2.0:1773941508.157776:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:2.0:1773941508.157777:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941508.157779:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941508.159128:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941508.159134:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941508.159136:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941508.159138:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941508.159141:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941508.159145:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941508.159149:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941508.159152:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941508.159154:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941508.159157:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324603d80 00000400:00000200:0.0:1773941508.159160:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941508.159163:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xb79 [1] + 0 00000800:00000001:0.0:1773941508.159167:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941508.159171:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941508.159173:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941508.159176:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941508.159180:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941508.159183:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06506300 x1860112291478912/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941518 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941508.159190:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d06506300 x1860112291478912/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941518 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941508.159203:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941508.159207:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941508.159209:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941508.159233:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941508.159237:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941508.159241:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941508.159243:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941508.159245:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941508.159250:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941508.159253:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941508.159255:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941508.159262:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06506300 x1860112291478912/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941518 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941508.159273:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06506300 x1860112291478912/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941518 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941508.159279:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941508.159280:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941508.159282:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941508.159283:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941508.159284:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941508.159286:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941508.159288:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941508.159291:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=9 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941508.159293:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941508.159295:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941508.159296:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941508.159298:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941508.159300:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941508.159302:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941508.159305:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06506300 x1860112291478912/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941518 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941508.159310:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941508.159311:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941508.159319:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941508.159321:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941508.159325:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941508.159326:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941508.159328:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941508.159329:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941508.159331:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941508.159333:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941508.159334:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941508.159337:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941508 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941508.159339:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941508.159345:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941508.159346:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941508.159348:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941508.159349:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941508.159350:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941508.159354:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941508.159355:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941508.159357:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941508.159360:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941508.159363:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941508.159365:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941508.159611:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941508.159617:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941508.159624:0:14721:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000040:2.0:1773941508.159626:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06506300 x1860112291478912/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941518 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1773941508.159633:0:14721:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:3.0:1773941508.159640:0:14721:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00100000:2.0:1773941508.159648:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06506300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291478912:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941508.159658:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941508.159661:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941508.159666:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06506300 x1860112291478912/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941518 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941508.159667:0:14721:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941508.159669:0:14721:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:3.0:1773941508.159673:0:14721:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941508.159677:0:14721:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941508.159681:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:3.0:1773941508.159683:0:14721:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941508.159685:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:3.0:1773941508.159686:0:14721:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941508.159689:0:14721:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941508.159690:0:14721:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941508.159690:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03520000. 10000000:00000001:3.0:1773941508.159694:0:14721:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941508.159697:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941508.159699:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941508.159701:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941508.159705:0:14721:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000001:2.0:1773941508.159706:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941508.159710:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d035a5c00. 00000100:00080000:3.0:1773941508.159711:0:14721:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 02000000:00000001:2.0:1773941508.159714:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1773941508.159715:0:14721:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:3.0:1773941508.159718:0:14721:0:(recover.c:147:ptlrpc_resend()) Process entered 02000000:00000001:2.0:1773941508.159718:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1773941508.159722:0:14721:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941508.159722:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06506300. 00000100:00080000:3.0:1773941508.159726:0:14721:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941508.159728:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941508.159729:0:14721:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941508.159733:0:14721:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000100:00000001:2.0:1773941508.159733:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000040:3.0:1773941508.159737:0:14721:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00000001:2.0:1773941508.159739:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941508.159744:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941508.159746:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941508.159749:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941508.159751:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1773941508.159756:0:14721:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:3.0:1773941508.159758:0:14721:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941508.159760:0:14721:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:3.0:1773941508.159763:0:14721:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:3.0:1773941508.159767:0:14721:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941508.159769:0:14721:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941508.159771:0:14721:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941508.159772:0:14721:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941508.159774:0:14721:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:2.1:1773941508.163002:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d071ee100. 00010000:00000010:2.1:1773941508.163010:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d089dfa00. 00000020:00000001:2.0:1773941509.115181:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941509.115212:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941509.115217:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941509.435078:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294899753, at 4294899776 00000100:00000010:1.0:1773941509.435098:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3300. 00000100:00000040:0.0:1773941509.435105:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294899753, at 4294899776 00000100:00000010:0.0:1773941509.435116:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3f00. 10000000:00000001:0.0:1773941509.755180:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1773941509.755190:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941509.755195:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941509.755197:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941509.755199:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941509.755200:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1773941509.755226:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:0.0:1773941509.755227:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:0.0:1773941509.755230:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:0.0:1773941509.755232:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1773941509.755234:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:0.0:1773941509.755243:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10836680. 00000020:00000040:0.0:1773941509.755248:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941509.755254:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941509.755258:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941509.755265:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941509.755274:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065c4e00. 00000100:00000001:0.0:1773941509.755283:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941509.755288:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1773941509.755292:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1773941509.755309:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d21ec1800. 00010000:00000001:0.0:1773941509.755317:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1773941509.755321:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d00031e00. 00000020:00000001:0.0:1773941509.755327:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:0.0:1773941509.755331:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d00031e00 with handle 0x8e86767622e290c to hash 00000020:00000001:0.0:1773941509.755333:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:0.0:1773941509.755334:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330090855936 : -113743618695680 : ffff988d00031e00) 00010000:00000001:0.0:1773941509.755343:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330090855936 : -113743618695680 : ffff988d00031e00) 00010000:00000001:0.0:1773941509.755348:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941509.755351:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941509.755354:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e290c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941509.755364:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e290c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941509.755377:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e290c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941509.755384:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1773941509.755399:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941509.755407:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb6a80. 00000100:00000001:0.0:1773941509.755410:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330139634304 : -113743569917312 : ffff988d02eb6a80) 00000100:00000001:0.0:1773941509.755414:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941509.755415:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367698560 : -113743341853056 : ffff988d10836680) 00000100:00000001:0.0:1773941509.755421:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941509.755427:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941509.755429:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941509.755432:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941509.755446:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10836680 x1860112291479040/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941509.755457:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941509.755460:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941509.755464:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941509.755466:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941509.755469:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941509.755471:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10836680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291479040:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941509.755475:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941509.755492:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941509.755495:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941509.755499:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941509.755504:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d02ce6800. 02000000:00000001:0.0:1773941509.755506:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941509.755516:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941509.755521:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0014bd10. 00000400:00000010:0.0:1773941509.755531:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833880. 00000100:00000200:0.0:1773941509.755541:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291479040, portal 25 00000100:00000001:0.0:1773941509.755543:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941509.755544:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367698560 : -113743341853056 : ffff988d10836680) 00000100:00000040:0.0:1773941509.755549:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10836680 x1860112291479040/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941525 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941509.755555:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941509.755557:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941509.755564:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833550. 00000100:00000200:0.0:1773941509.755567:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291479040, offset 0 00000400:00000200:0.0:1773941509.755573:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941509.755582:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941509.755590:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941509.755595:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941509.755599:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941509.755602:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941509.755605:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941509.755607:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[20, 19] 00000400:00000200:0.0:1773941509.755610:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:0.0:1773941509.755613:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [131:131:256:4294967295] 192.168.204.124@tcp LPNI seq info [21:21:8:4294967295] 00000400:00000200:0.0:1773941509.755616:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941509.755625:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:0.0:1773941509.755630:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:0.0:1773941509.755635:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a20bd00. 00000800:00000200:0.0:1773941509.755641:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:0.0:1773941509.755650:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:0.0:1773941509.755657:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a20bd00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1773941509.755683:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941509.755689:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941509.755693:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294901351 00000100:00000010:0.0:1773941509.755698:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580b40. 00000100:00000001:0.0:1773941509.755708:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1773941509.755716:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294901351 00000100:00000010:0.0:1773941509.755721:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d065803c0. 00000100:00000001:0.0:1773941509.755725:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1773941509.755730:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941509.755733:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941509.755736:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941509.755745:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1773941509.755750:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6a80 going to sleep for 11 seconds 00000100:00000001:0.0:1773941509.755753:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941509.755757:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941509.755815:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941509.755821:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a20bd00. 00000400:00000200:1.0:1773941509.755831:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941509.755836:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941509.755842:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833550 00000400:00000010:1.0:1773941509.755844:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833550. 00000100:00000001:1.0:1773941509.755849:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941509.755856:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10836680 x1860112291479040/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941525 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941509.755868:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941509.755872:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d10836680 x1860112291479040/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941525 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941509.755881:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941509.755884:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941509.756958:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941509.756992:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941509.756995:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941509.756998:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941509.757005:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941509.757021:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941509.757028:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941509.757032:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941509.757037:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941509.757042:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324603e00 00000400:00000200:0.0:1773941509.757047:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941509.757054:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xb81 [1] + 224 00000800:00000001:0.0:1773941509.757062:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941509.757070:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941509.757073:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941509.757077:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941509.757082:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941509.757088:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10836680 x1860112291479040/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941525 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941509.757107:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d10836680 x1860112291479040/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941525 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941509.757135:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941509.757143:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941509.757148:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941509.757183:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941509.757192:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941509.757193:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941509.757196:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0014bd10. 00000400:00000200:2.0:1773941509.757200:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833880 00000400:00000010:2.0:1773941509.757201:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833880. 00000100:00000001:2.0:1773941509.757203:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941509.757208:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10836680 x1860112291479040/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941525 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941509.757217:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10836680 x1860112291479040/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941525 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941509.757222:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941509.757224:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941509.757226:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941509.757229:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941509.757231:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941509.757233:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941509.757235:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941509.757241:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=10 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941509.757244:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941509.757247:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:2.0:1773941509.757251:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:2.0:1773941509.761762:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941509.761769:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:2.0:1773941509.761775:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1773941509.761780:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1773941509.761785:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1773941509.766702:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941509.766709:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941509.766711:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941509.766735:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:2.0:1773941509.766748:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d10836680 x1860112291479040/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941525 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941509.766760:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941509.766762:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941509.766765:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941509.766769:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941509.766773:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:2.0:1773941509.766775:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941509.766777:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941509.766778:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941509.766780:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941509.766781:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941509.766783:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941509.766787:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1773941509.766790:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1773941509.766795:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941509.766799:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 231 00000400:00000200:2.0:1773941509.766810:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941509.766818:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:2.0:1773941509.766823:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941509.766827:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941509.766830:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941509.766833:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:2.0:1773941509.766837:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941509.766840:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941509.766844:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941509.766847:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941509.766851:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941509.766854:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:2.0:1773941509.766858:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941509.766860:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941509.766862:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941509.766864:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:2.0:1773941509.766875:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:2.0:1773941509.766880:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:2.0:1773941509.766883:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941509.766886:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941509.766921:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941509.766925:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941509.766928:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941509.766930:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941509.766938:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06506300. 00000020:00000040:2.0:1773941509.766942:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:2.0:1773941509.766947:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941509.766949:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941509.766952:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941509.766955:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941509.766983:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941509.766993:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d035a5c00. 00000100:00000001:2.0:1773941509.767002:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941509.767079:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d06506300 x1860112291479168/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941509.767088:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941509.767092:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941509.767096:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005464cf12] to pc [ptlrpcd_rcv+-1] req@ffff988d06506300 x1860112291479168/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941509.767118:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941509.767122:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:2.0:1773941509.767125:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941509.767130:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941509.767135:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941509.767138:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941509.767142:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6a80 going to sleep for 1 seconds 00000100:00000001:2.0:1773941509.767146:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941509.767150:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941509.767156:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10836680 x1860112291479040/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941525 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941509.767176:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941509.767180:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941509.767183:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941509.767184:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941509.767189:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941509.767194:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1773941509.767195:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10836680 x1860112291479040/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941525 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941509.767198:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00100000:2.0:1773941509.767210:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10836680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291479040:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000040:3.0:1773941509.767217:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06506300 x1860112291479168/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941509.767219:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941509.767231:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941509.767241:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:2.0:1773941509.767247:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1773941509.767249:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941509.767252:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941509.767254:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10836680 x1860112291479040/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941525 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1773941509.767257:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941509.767264:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941509.767267:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941509.767268:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941509.767271:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb6a80. 00000100:00100000:3.0:1773941509.767276:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06506300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291479168:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941509.767281:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1773941509.767281:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:2.0:1773941509.767289:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:3.0:1773941509.767291:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941509.767295:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941509.767296:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:2.0:1773941509.767304:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 02000000:00000001:3.0:1773941509.767305:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:2.0:1773941509.767313:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941509.767316:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00031e00 refcount=4 00000020:00000001:2.0:1773941509.767318:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090855936 : -113743618695680 : ffff988d00031e00) 02000000:00000010:3.0:1773941509.767322:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03522800. 00010000:00000001:2.0:1773941509.767322:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090855936 : -113743618695680 : ffff988d00031e00) 00010000:00010000:2.0:1773941509.767328:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e290c lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941509.767336:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00010000:00010000:2.0:1773941509.767339:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e290c lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941509.767350:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 02000000:00000001:3.0:1773941509.767357:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941509.767358:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e290c lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941509.767374:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941509.767379:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:3.0:1773941509.767380:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:2.0:1773941509.767384:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e290c lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:3.0:1773941509.767388:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00010000:00000001:2.0:1773941509.767401:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000010:3.0:1773941509.767402:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00010000:00000001:2.0:1773941509.767406:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1773941509.767413:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291479168, portal 25 00010000:00000001:2.0:1773941509.767413:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000100:00000001:3.0:1773941509.767421:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00010000:2.0:1773941509.767422:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e290c lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941509.767423:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196583168 : -113743512968448 : ffff988d06506300) 00000100:00000040:3.0:1773941509.767435:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06506300 x1860112291479168/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941519 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:2.0:1773941509.767442:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d00031e00 already unused, calling callback (ffffffffc12b4c20) 00000100:00000001:3.0:1773941509.767446:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941509.767452:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 10000000:00000001:2.0:1773941509.767452:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000010:3.0:1773941509.767460:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac550. 10000000:00010000:2.0:1773941509.767461:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e290c lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:3.0:1773941509.767464:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291479168, offset 0 00000400:00000200:3.0:1773941509.767480:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00000001:2.0:1773941509.767488:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:2.0:1773941509.767495:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:2.0:1773941509.767498:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941509.767500:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00031e00 refcount=5 00000400:00000200:3.0:1773941509.767501:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000020:00000001:2.0:1773941509.767511:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090855936 : -113743618695680 : ffff988d00031e00) 00000400:00000200:3.0:1773941509.767518:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:2.0:1773941509.767520:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090855936 : -113743618695680 : ffff988d00031e00) 00000400:00000200:3.0:1773941509.767526:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:2.0:1773941509.767528:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:3.0:1773941509.767532:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00010000:2.0:1773941509.767535:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e290c lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941509.767541:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941509.767552:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941509.767567:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[20, 21] 10000000:00000001:2.0:1773941509.767568:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:3.0:1773941509.767578:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 10000000:00010000:2.0:1773941509.767579:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e290c lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1773941509.767590:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:3.0:1773941509.767591:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [132:132:256:4294967295] 192.168.204.114@tcp LPNI seq info [22:22:8:4294967295] 10000000:00000040:2.0:1773941509.767595:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:2.0:1773941509.767597:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941509.767600:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:2.0:1773941509.767602:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:2.0:1773941509.767605:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1773941509.767608:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1773941509.767610:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941509.767612:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941509.767616:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000020:00000040:2.0:1773941509.767617:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d00031e00 with handle 0x8e86767622e290c from hash 00010000:00000001:2.0:1773941509.767623:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1773941509.767625:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941509.767628:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941509.767631:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1773941509.767633:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:2.0:1773941509.767637:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:2.0:1773941509.767642:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941509.767647:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941509.767649:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941509.767653:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941509.767655:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00010000:00010000:2.0:1773941509.767658:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e290c lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941509.767669:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941509.767671:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941509.767675:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000800:00000200:3.0:1773941509.767678:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00010000:00000001:2.0:1773941509.767680:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:2.0:1773941509.767683:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000010:3.0:1773941509.767684:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74500. 00010000:00000001:2.0:1773941509.767685:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941509.767691:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1773941509.767694:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e290c lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941509.767701:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00010000:00000040:2.0:1773941509.767716:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d21ec1800 count: 0 00000800:00000200:3.0:1773941509.767718:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:3.0:1773941509.767722:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74500 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:2.0:1773941509.767742:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:3.0:1773941509.767745:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941509.767746:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941509.767751:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941509.767758:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10836680 x1860112291479040/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941525 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941509.767759:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:3.0:1773941509.767768:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294901363 00000100:00000001:2.0:1773941509.767781:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1773941509.767784:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294901351 02000000:00000001:2.0:1773941509.767785:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941509.767790:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d02ce6800. 00000100:00000001:3.0:1773941509.767793:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294901363 02000000:00000001:2.0:1773941509.767795:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941509.767798:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:3.0:1773941509.767799:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294901351 00000100:00000001:3.0:1773941509.767806:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000040:2.0:1773941509.767806:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941509.767815:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941509.767816:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941509.767819:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065c4e00. 00000100:00000001:3.0:1773941509.767823:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941509.767825:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941509.767829:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941509.767832:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10836680. 00000100:00000001:3.0:1773941509.767833:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941509.767840:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941509.767842:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:2.0:1773941509.767844:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941509.767847:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 10000000:01000000:2.0:1773941509.767847:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:2.0:1773941509.767853:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000100:00000001:3.0:1773941509.767854:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 10000000:00000010:2.0:1773941509.767858:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d05437dc0. 00000100:00000001:3.0:1773941509.767860:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000010:2.0:1773941509.767864:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d0381e000. 00000100:00000001:3.0:1773941509.767865:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941509.767870:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1773941509.767871:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d0381e800. 00000100:00000001:3.0:1773941509.767879:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000040:00000010:2.0:1773941509.767879:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d0381f200. 00000020:00000010:2.0:1773941509.767888:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941509.767895:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d03356200. 00000800:00000200:1.0:1773941509.767898:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 80000000:00000010:2.0:1773941509.767900:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d08bbb420. 00000800:00000010:1.0:1773941509.767908:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74500. 00040000:00000010:2.0:1773941509.767909:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d08a4f400. 20000000:00000010:2.0:1773941509.767915:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d08a4fe00. 00000400:00000200:1.0:1773941509.767918:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 10000000:00000040:2.0:1773941509.767920:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:2.0:1773941509.767924:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:2.0:1773941509.767929:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d0381e800. 00000400:00000200:1.0:1773941509.767933:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000040:00000010:2.0:1773941509.767936:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d0381f200. 00000400:00000200:1.0:1773941509.767942:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac550 00000020:00000010:2.0:1773941509.767945:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d09305000. 00000400:00000010:1.0:1773941509.767947:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac550. 00000020:00000010:2.0:1773941509.767951:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d03356200. 00000100:00000001:1.0:1773941509.767954:0:2393:0:(events.c:34:request_out_callback()) Process entered 80000000:00000010:2.0:1773941509.767955:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d08bbb420. 00040000:00000010:2.0:1773941509.767994:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d08a4f400. 00000100:00000200:1.0:1773941509.767999:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06506300 x1860112291479168/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941519 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 20000000:00000010:2.0:1773941509.768000:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d08a4fe00. 00000020:00000010:2.0:1773941509.768005:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0381e000. 10000000:00000010:2.0:1773941509.768024:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d05437dc0. 10000000:01000000:2.0:1773941509.768029:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:2.0:1773941509.768035:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941509.768037:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 10000000:00020000:2.0:1773941509.768039:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:2.0:1773941509.768042:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 00000100:00000040:1.0:1773941509.768045:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06506300 x1860112291479168/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941519 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000001:2.0:1773941509.768046:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:2.0:1773941509.768048:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941509.768053:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000100:00000001:1.0:1773941509.768059:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941509.768062:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000010:2.1:1773941509.773037:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d21ec1800. 00010000:00000010:2.1:1773941509.773053:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d00031e00. 00000800:00000001:0.0:1773941509.774196:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941509.774216:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941509.774220:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941509.774223:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941509.774232:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941509.774240:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941509.774261:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941509.774267:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941509.774273:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941509.774277:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324603e80 00000400:00000200:0.0:1773941509.774281:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941509.774285:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xb91 [1] + 0 00000800:00000001:0.0:1773941509.774292:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941509.774299:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941509.774301:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941509.774305:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941509.774311:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941509.774318:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06506300 x1860112291479168/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941519 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941509.774327:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d06506300 x1860112291479168/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941519 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941509.774343:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941509.774352:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941509.774356:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941509.774375:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941509.774379:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941509.774387:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941509.774388:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941509.774391:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941509.774395:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941509.774397:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941509.774399:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941509.774403:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06506300 x1860112291479168/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941519 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941509.774410:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06506300 x1860112291479168/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941519 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941509.774414:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941509.774416:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941509.774418:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941509.774420:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941509.774422:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941509.774424:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941509.774426:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941509.774431:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=10 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941509.774440:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941509.774443:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941509.774444:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941509.774446:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941509.774448:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941509.774450:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941509.774454:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06506300 x1860112291479168/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941519 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941509.774459:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941509.774461:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941509.774463:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941509.774465:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941509.774470:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941509.774473:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941509.774475:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941509.774477:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941509.774479:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941509.774480:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941509.774482:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941509.774484:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941509 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941509.774486:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941509.774491:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941509.774493:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941509.774496:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941509.774498:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941509.774499:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941509.774502:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941509.774503:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941509.774506:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941509.774508:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941509.774510:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941509.774513:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941509.774743:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941509.774750:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941509.774763:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06506300 x1860112291479168/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941519 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941509.774774:0:14722:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:2.0:1773941509.774778:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06506300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291479168:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00080000:3.0:1773941509.774789:0:14722:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00000001:2.0:1773941509.774790:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1773941509.774794:0:14722:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:3.0:1773941509.774798:0:14722:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:2.0:1773941509.774798:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 10000000:01000000:3.0:1773941509.774800:0:14722:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:3.0:1773941509.774804:0:14722:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941509.774809:0:14722:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000040:2.0:1773941509.774810:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06506300 x1860112291479168/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941519 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000001:00000001:3.0:1773941509.774813:0:14722:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941509.774820:0:14722:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941509.774822:0:14722:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941509.774825:0:14722:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941509.774833:0:14722:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941509.774842:0:14722:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941509.774849:0:14722:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941509.774850:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1773941509.774858:0:14722:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 02000000:00000001:2.0:1773941509.774860:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1773941509.774862:0:14722:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941509.774867:0:14722:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1773941509.774869:0:14722:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 02000000:00000010:2.0:1773941509.774870:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03522800. 00000100:00000001:3.0:1773941509.774872:0:14722:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941509.774876:0:14722:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:3.0:1773941509.774879:0:14722:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 02000000:00000001:2.0:1773941509.774881:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941509.774886:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941509.774888:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941509.774892:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941509.774895:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d035a5c00. 00000100:00080000:3.0:1773941509.774899:0:14722:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 02000000:00000001:2.0:1773941509.774901:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 10000000:00000001:3.0:1773941509.774904:0:14722:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941509.774907:0:14722:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:3.0:1773941509.774911:0:14722:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 02000000:00000001:2.0:1773941509.774920:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941509.774926:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06506300. 00000100:00000001:2.0:1773941509.774932:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941509.774936:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941509.774940:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941509.774951:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941509.774954:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941509.774957:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941509.774985:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941509.778454:0:14722:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941509.778459:0:14722:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941509.778461:0:14722:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941509.778464:0:14722:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941509.778465:0:14722:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941510.139165:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941510.139175:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941510.139183:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941510.395156:0:461:0:(pinger.c:229:ptlrpc_pinger_process_import()) c1c1ff21-5f68-4bf6-bdc4-258644b24465->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1773941510.395172:0:461:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1773941510.395187:0:461:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c2300. 00000020:00000040:1.0:1773941510.395195:0:461:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941510.395200:0:461:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941510.395204:0:461:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941510.395231:0:461:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941510.395242:0:461:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff988d02cfd800. 00000100:00000001:1.0:1773941510.395252:0:461:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941510.395267:0:461:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging c1c1ff21-5f68-4bf6-bdc4-258644b24465->MGS req@ffff988d203c2300 x1860112291479296/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941510.395281:0:461:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941510.395287:0:461:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941510.395296:0:461:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093c3ab5c] to pc [ptlrpcd_00_00+0] req@ffff988d203c2300 x1860112291479296/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941510.395349:0:461:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941510.395356:0:461:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (238) 00000100:00000001:1.0:1773941510.395393:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1773941510.395399:0:2402:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:0.0:1773941510.395402:0:2401:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941510.395403:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941510.395412:0:2401:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941510.395417:0:2402:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941510.395422:0:2402:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941510.395425:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941510.395427:0:2402:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941510.395431:0:2402:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941510.395439:0:2402:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c2300 x1860112291479296/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941510.395449:0:2402:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941510.395452:0:2402:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941510.395457:0:2402:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941510.395461:0:2402:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941510.395464:0:2402:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941510.395469:0:2402:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2402:1860112291479296:192.168.204.114@tcp:400:kworker.0 00000100:00000001:1.0:1773941510.395474:0:2402:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941510.395481:0:2402:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941510.395484:0:2402:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941510.395487:0:2402:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941510.395493:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff988d181fc200. 02000000:00000001:1.0:1773941510.395497:0:2402:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941510.395506:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941510.395519:0:2402:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c5d8. 00000400:00000010:1.0:1773941510.395529:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000100:00000200:1.0:1773941510.395539:0:2402:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112291479296, portal 25 00000100:00000001:1.0:1773941510.395541:0:2402:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941510.395543:0:2402:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631463680 : -113743078087936 : ffff988d203c2300) 00000100:00000040:1.0:1773941510.395551:0:2402:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c2300 x1860112291479296/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941526 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941510.395560:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941510.395563:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941510.395567:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113decc0. 00000100:00000200:1.0:1773941510.395570:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1860112291479296, offset 0 00000400:00000200:1.0:1773941510.395578:0:2402:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941510.395588:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941510.395598:0:2402:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941510.395604:0:2402:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941510.395610:0:2402:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941510.395615:0:2402:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941510.395618:0:2402:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941510.395622:0:2402:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[22, 21] 00000400:00000200:1.0:1773941510.395626:0:2402:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:1.0:1773941510.395631:0:2402:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [133:133:256:4294967295] 192.168.204.124@tcp LPNI seq info [23:23:8:4294967295] 00000400:00000200:1.0:1773941510.395636:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941510.395647:0:2402:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:1.0:1773941510.395654:0:2402:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:1.0:1773941510.395660:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfda00. 00000800:00000200:1.0:1773941510.395670:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:1.0:1773941510.395679:0:2402:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:1.0:1773941510.395686:0:2402:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfda00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1773941510.395705:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941510.395710:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941510.395712:0:2402:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294901991 00000100:00000001:1.0:1773941510.395730:0:2402:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294901363 00000100:00000001:1.0:1773941510.395737:0:2402:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294901991 00000100:00000001:1.0:1773941510.395741:0:2402:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294901363 00000100:00000001:1.0:1773941510.395745:0:2402:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941510.395748:0:2402:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941510.395753:0:2402:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941510.395756:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941510.395832:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941510.395837:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfda00. 00000400:00000200:0.0:1773941510.395842:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941510.395849:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941510.395855:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113decc0 00000400:00000010:0.0:1773941510.395857:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113decc0. 00000100:00000001:0.0:1773941510.395861:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941510.395867:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c2300 x1860112291479296/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941526 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941510.395877:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941510.395880:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c2300 x1860112291479296/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941526 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941510.395901:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941510.395903:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941510.396940:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941510.396988:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941510.396991:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941510.396993:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941510.397011:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941510.397015:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941510.397021:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941510.397024:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941510.397028:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941510.397031:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324603f00 00000400:00000200:0.0:1773941510.397035:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941510.397039:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xb99 [1] + 224 00000800:00000001:0.0:1773941510.397046:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941510.397054:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941510.397055:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941510.397058:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941510.397061:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941510.397064:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c2300 x1860112291479296/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941526 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941510.397073:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff988d203c2300 x1860112291479296/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941526 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941510.397090:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941510.397096:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941510.397098:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941510.397134:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941510.397138:0:2402:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941510.397145:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941510.397147:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941510.397150:0:2402:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:1.0:1773941510.397155:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:1.0:1773941510.397158:0:2402:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:1.0:1773941510.397161:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941510.397168:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c2300 x1860112291479296/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941526 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941510.397180:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c2300 x1860112291479296/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941526 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941510.397188:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941510.397190:0:2402:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941510.397192:0:2402:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941510.397195:0:2402:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941510.397198:0:2402:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941510.397201:0:2402:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941510.397204:0:2402:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941510.397211:0:2402:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=11 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941510.397215:0:2402:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941510.397218:0:2402:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1773941510.397220:0:2402:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941510.397223:0:2402:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941510.397227:0:2402:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941510.397231:0:2402:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941510.397245:0:2402:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941510.397248:0:2402:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941510.397251:0:2402:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941510.397253:0:2402:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941510.397256:0:2402:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941510.397260:0:2402:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d203c2300 x1860112291479296/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941526 ref 1 fl Rpc:RNQU/200/0 rc 0/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941510.397269:0:2402:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941510.397270:0:2402:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941510.397272:0:2402:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941510.397276:0:2402:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941510.397279:0:2402:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941510.397281:0:2402:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941510.397283:0:2402:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941510.397285:0:2402:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941510.397286:0:2402:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941510.397288:0:2402:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941510.397290:0:2402:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941510.397293:0:2402:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941510.397295:0:2402:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941510.397299:0:2402:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941510.397303:0:2402:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 232 00000400:00000200:1.0:1773941510.397309:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941510.397313:0:2402:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941510.397318:0:2402:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941510.397321:0:2402:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941510.397324:0:2402:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941510.397327:0:2402:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941510.397330:0:2402:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941510.397333:0:2402:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941510.397336:0:2402:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941510.397339:0:2402:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941510.397342:0:2402:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941510.397351:0:2402:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941510.397354:0:2402:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941510.397357:0:2402:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941510.397359:0:2402:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941510.397361:0:2402:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941510.397364:0:2402:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941510.397368:0:2402:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941510.397371:0:2402:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941510.397374:0:2402:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941510.397378:0:2402:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941510.397381:0:2402:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941510.397383:0:2402:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941510.397386:0:2402:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941510.397390:0:2402:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c1880. 00000020:00000040:1.0:1773941510.397393:0:2402:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941510.397397:0:2402:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941510.397399:0:2402:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941510.397402:0:2402:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941510.397404:0:2402:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941510.397407:0:2402:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941510.397412:0:2402:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d1118b000. 00000100:00000001:1.0:1773941510.397416:0:2402:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941510.397422:0:2402:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c1880 x1860112291479424/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941510.397430:0:2402:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941510.397433:0:2402:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941510.397437:0:2402:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008b2304da] to pc [ptlrpcd_rcv+-1] req@ffff988d203c1880 x1860112291479424/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941510.397458:0:2402:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941510.397461:0:2402:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941510.397463:0:2402:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:1.0:1773941510.397468:0:2402:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c2300 x1860112291479296/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941526 ref 1 fl Rpc:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941510.397481:0:2402:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941510.397485:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941510.397488:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941510.397495:0:2402:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c2300 x1860112291479296/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941526 ref 1 fl Interpret:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941510.397510:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1773941510.397510:0:2402:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2402:1860112291479296:192.168.204.114@tcp:400:kworker.0 00000100:00000001:2.0:1773941510.397517:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941510.397521:0:2402:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941510.397522:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941510.397527:0:2402:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941510.397536:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c1880 x1860112291479424/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941510.397537:0:2402:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c2300 x1860112291479296/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941526 ref 1 fl Complete:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941510.397607:0:2402:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1773941510.397608:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941510.397614:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941510.397619:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941510.397622:0:2402:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1773941510.397623:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000010:1.0:1773941510.397628:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff988d181fc200. 02000000:00000001:2.0:1773941510.397629:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941510.397641:0:2402:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00100000:2.0:1773941510.397643:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291479424:192.168.204.114@tcp:250:ptlrpcd_00_01.0 00000020:00000001:1.0:1773941510.397645:0:2402:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941510.397650:0:2402:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941510.397652:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000001:1.0:1773941510.397657:0:2402:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000001:2.0:1773941510.397659:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000010:1.0:1773941510.397663:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff988d02cfd800. 02000000:00000001:2.0:1773941510.397664:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941510.397670:0:2402:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941510.397671:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1773941510.397677:0:2402:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941510.397680:0:2402:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c2300. 02000000:00000010:2.0:1773941510.397681:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a4800. 02000000:00000001:2.0:1773941510.397688:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941510.397691:0:2402:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941510.397695:0:2402:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941510.397697:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941510.397700:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1773941510.397703:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000100:00000001:1.0:1773941510.397709:0:2402:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000400:00000010:2.0:1773941510.397712:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000001:1.0:1773941510.397713:0:2402:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941510.397722:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941510.397725:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1773941510.397726:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291479424, portal 25 00000100:00000001:2.0:1773941510.397734:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941510.397736:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631460992 : -113743078090624 : ffff988d203c1880) 00000100:00000040:2.0:1773941510.397752:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c1880 x1860112291479424/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941520 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941510.397761:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941510.397764:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:2.0:1773941510.397767:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516220. 00000100:00000200:2.0:1773941510.397771:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291479424, offset 0 00000400:00000200:2.0:1773941510.397777:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941510.397783:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941510.397790:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941510.397795:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941510.397799:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941510.397802:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941510.397805:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941510.397809:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[22, 23] 00000400:00000200:2.0:1773941510.397813:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:2.0:1773941510.397818:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [134:134:256:4294967295] 192.168.204.114@tcp LPNI seq info [24:24:8:4294967295] 00000400:00000200:2.0:1773941510.397823:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941510.397832:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:2.0:1773941510.397838:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:2.0:1773941510.397843:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0db600. 00000800:00000200:2.0:1773941510.397849:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:2.0:1773941510.397856:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:2.0:1773941510.397861:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0db600 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:2.0:1773941510.397876:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941510.397880:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941510.397883:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294901993 00000100:00000001:2.0:1773941510.397891:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294901991 00000100:00000001:2.0:1773941510.397896:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294901993 00000100:00000001:2.0:1773941510.397899:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294901991 00000100:00000001:2.0:1773941510.397902:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941510.397910:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941510.397913:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941510.397916:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941510.397924:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941510.397926:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941510.397929:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941510.397931:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941510.397933:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941510.397936:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941510.398007:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941510.398011:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0db600. 00000400:00000200:0.0:1773941510.398015:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941510.398019:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941510.398022:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516220 00000400:00000010:0.0:1773941510.398030:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516220. 00000100:00000001:0.0:1773941510.398032:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941510.398036:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c1880 x1860112291479424/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941520 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941510.398043:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941510.398046:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c1880 x1860112291479424/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941520 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941510.398051:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941510.398052:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941510.399498:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941510.399505:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941510.399508:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941510.399510:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941510.399514:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941510.399521:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941510.399525:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941510.399528:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941510.399531:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941510.399534:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324603f80 00000400:00000200:0.0:1773941510.399537:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941510.399542:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xba9 [1] + 0 00000800:00000001:0.0:1773941510.399546:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941510.399551:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941510.399553:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941510.399555:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941510.399558:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941510.399561:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c1880 x1860112291479424/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941520 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941510.399568:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c1880 x1860112291479424/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941520 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941510.399581:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941510.399590:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941510.399592:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941510.399620:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941510.399623:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941510.399628:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941510.399630:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941510.399633:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941510.399637:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941510.399638:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941510.399641:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941510.399647:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c1880 x1860112291479424/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941520 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941510.399657:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c1880 x1860112291479424/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941520 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941510.399665:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941510.399667:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941510.399670:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941510.399672:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941510.399675:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941510.399677:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941510.399680:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941510.399686:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=11 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941510.399690:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941510.399693:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941510.399695:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941510.399697:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941510.399700:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941510.399703:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941510.399707:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c1880 x1860112291479424/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941520 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941510.399727:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941510.399730:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941510.399732:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941510.399734:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941510.399740:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941510.399743:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941510.399745:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941510.399748:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941510.399751:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941510.399753:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941510.399756:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:2.0:1773941510.399759:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941510 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941510.399763:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941510.399769:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:2.0:1773941510.399772:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941510.399775:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941510.399778:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941510.399780:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941510.399783:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:2.0:1773941510.399785:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941510.399788:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941510.399792:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941510.399795:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:2.0:1773941510.399798:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941510.400015:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941510.400024:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941510.400024:0:14723:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000040:3.0:1773941510.400038:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c1880 x1860112291479424/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941520 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1773941510.400039:0:14723:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:2.0:1773941510.400052:0:14723:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00100000:3.0:1773941510.400058:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291479424:192.168.204.114@tcp:250:ptlrpcd_00_01.0 00000100:00000001:3.0:1773941510.400079:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941510.400082:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941510.400086:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c1880 x1860112291479424/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941520 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941510.400092:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941510.400094:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941510.400097:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a4800. 00000100:00000001:2.0:1773941510.400099:0:14723:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:3.0:1773941510.400101:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941510.400106:0:2400:0:(genops.c:1236:class_import_put()) Process entered 10000000:01000000:2.0:1773941510.400106:0:14723:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000020:00000040:3.0:1773941510.400111:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941510.400115:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:2.0:1773941510.400116:0:14723:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000010:3.0:1773941510.400118:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d1118b000. 00000001:00000001:2.0:1773941510.400122:0:14723:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:3.0:1773941510.400123:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941510.400125:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941510.400126:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c1880. 00000100:00000001:3.0:1773941510.400129:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941510.400133:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941510.400135:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:2.0:1773941510.400137:0:14723:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941510.400139:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941510.400144:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:2.0:1773941510.400146:0:14723:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:3.0:1773941510.400149:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000001:00000001:2.0:1773941510.400150:0:14723:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:3.0:1773941510.400151:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:2.0:1773941510.400155:0:14723:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941510.400159:0:14723:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941510.400163:0:14723:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941510.400168:0:14723:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941510.400171:0:14723:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941510.400174:0:14723:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:2.0:1773941510.400176:0:14723:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1773941510.400178:0:14723:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941510.400181:0:14723:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941510.400182:0:14723:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:2.0:1773941510.400185:0:14723:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:2.0:1773941510.400203:0:14723:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:2.0:1773941510.400204:0:14723:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941510.400206:0:14723:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:2.0:1773941510.400211:0:14723:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:2.0:1773941510.400215:0:14723:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941510.400218:0:14723:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941510.400221:0:14723:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941510.400224:0:14723:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941510.400226:0:14723:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941511.163124:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941511.163135:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941511.163142:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:0.0:1773941511.419103:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1773941511.419112:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941511.419116:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941511.419119:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941511.419120:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941511.419122:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1773941511.419125:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:0.0:1773941511.419127:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:0.0:1773941511.419129:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:0.0:1773941511.419131:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1773941511.419133:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:0.0:1773941511.419147:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10835f80. 00000020:00000040:0.0:1773941511.419152:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941511.419157:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941511.419159:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941511.419166:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941511.419174:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065c5400. 00000100:00000001:0.0:1773941511.419181:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941511.419186:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1773941511.419189:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1773941511.419197:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d21ec1400. 00010000:00000001:0.0:1773941511.419204:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1773941511.419207:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d00030600. 00000020:00000001:0.0:1773941511.419212:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:0.0:1773941511.419214:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d00030600 with handle 0x8e86767622e2913 to hash 00000020:00000001:0.0:1773941511.419215:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:0.0:1773941511.419217:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330090849792 : -113743618701824 : ffff988d00030600) 00010000:00000001:0.0:1773941511.419219:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330090849792 : -113743618701824 : ffff988d00030600) 00010000:00000001:0.0:1773941511.419224:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941511.419226:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941511.419229:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e2913 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941511.419239:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e2913 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941511.419271:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e2913 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941511.419278:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1773941511.419281:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941511.419291:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb6a80. 00000100:00000001:0.0:1773941511.419303:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330139634304 : -113743569917312 : ffff988d02eb6a80) 00000100:00000001:0.0:1773941511.419306:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941511.419308:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367696768 : -113743341854848 : ffff988d10835f80) 00000100:00000001:0.0:1773941511.419315:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941511.419321:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941511.419324:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941511.419326:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941511.419338:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10835f80 x1860112291479552/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941511.419345:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941511.419348:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941511.419351:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941511.419354:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941511.419356:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941511.419359:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10835f80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291479552:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941511.419362:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941511.419368:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941511.419370:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941511.419372:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941511.419377:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d02ce7000. 02000000:00000001:0.0:1773941511.419378:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941511.419387:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941511.419390:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0014b580. 00000400:00000010:0.0:1773941511.419397:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d108334c8. 00000100:00000200:0.0:1773941511.419405:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291479552, portal 25 00000100:00000001:0.0:1773941511.419407:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941511.419413:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367696768 : -113743341854848 : ffff988d10835f80) 00000100:00000040:0.0:1773941511.419417:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10835f80 x1860112291479552/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941527 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941511.419424:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941511.419426:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941511.419428:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833440. 00000100:00000200:0.0:1773941511.419431:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291479552, offset 0 00000400:00000200:0.0:1773941511.419436:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941511.419445:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941511.419452:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941511.419457:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941511.419461:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941511.419472:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941511.419474:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941511.419476:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[24, 23] 00000400:00000200:0.0:1773941511.419479:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:0.0:1773941511.419483:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [135:135:256:4294967295] 192.168.204.124@tcp LPNI seq info [25:25:8:4294967295] 00000400:00000200:0.0:1773941511.419487:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941511.419495:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:0.0:1773941511.419500:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:0.0:1773941511.419505:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a20bd00. 00000800:00000200:0.0:1773941511.419512:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:0.0:1773941511.419519:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:0.0:1773941511.419524:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a20bd00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1773941511.419543:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941511.419546:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941511.419550:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294903015 00000100:00000001:0.0:1773941511.419557:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294901993 00000100:00000001:0.0:1773941511.419562:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294903015 00000100:00000001:0.0:1773941511.419569:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294901993 00000100:00000001:0.0:1773941511.419577:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941511.419579:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941511.419582:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941511.419584:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1773941511.419587:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6a80 going to sleep for 11 seconds 00000100:00000001:0.0:1773941511.419590:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941511.419593:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941511.419677:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941511.419682:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a20bd00. 00000400:00000200:1.0:1773941511.419687:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941511.419693:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941511.419705:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833440 00000400:00000010:1.0:1773941511.419707:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833440. 00000100:00000001:1.0:1773941511.419711:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941511.419715:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10835f80 x1860112291479552/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941527 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941511.419724:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941511.419727:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d10835f80 x1860112291479552/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941527 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941511.419733:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941511.419734:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941511.420678:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941511.420691:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941511.420695:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941511.420698:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941511.420704:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941511.420711:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941511.420719:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941511.420724:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941511.420729:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941511.420735:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324604000 00000400:00000200:0.0:1773941511.420740:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941511.420748:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xbb1 [1] + 224 00000800:00000001:0.0:1773941511.420756:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941511.420765:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941511.420768:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941511.420772:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941511.420778:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941511.420783:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10835f80 x1860112291479552/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941527 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941511.420796:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d10835f80 x1860112291479552/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941527 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941511.420818:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941511.420825:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941511.420829:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941511.420874:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941511.420884:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941511.420886:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941511.420890:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0014b580. 00000400:00000200:2.0:1773941511.420895:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d108334c8 00000400:00000010:2.0:1773941511.420897:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d108334c8. 00000100:00000001:2.0:1773941511.420901:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941511.420907:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10835f80 x1860112291479552/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941527 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941511.420919:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10835f80 x1860112291479552/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941527 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941511.420939:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941511.420942:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941511.420946:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941511.420949:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941511.420952:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941511.420954:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941511.420957:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941511.421039:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=12 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941511.421044:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941511.421047:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:2.0:1773941511.421052:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:2.0:1773941511.421056:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941511.421060:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:2.0:1773941511.421063:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1773941511.421068:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1773941511.421073:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1773941511.421077:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941511.421080:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941511.421082:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941511.421085:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:2.0:1773941511.421090:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d10835f80 x1860112291479552/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941527 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941511.421098:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941511.421100:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941511.421102:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941511.421105:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941511.421108:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:2.0:1773941511.421110:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941511.421112:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941511.421113:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941511.421114:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941511.421116:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941511.421124:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941511.421127:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1773941511.421129:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1773941511.421133:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941511.421136:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 233 00000400:00000200:2.0:1773941511.421142:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941511.421147:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:2.0:1773941511.421152:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941511.421156:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941511.421159:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941511.421162:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:2.0:1773941511.421166:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941511.421169:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941511.421172:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941511.421175:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941511.421177:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941511.421180:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:2.0:1773941511.421183:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941511.421185:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941511.421187:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941511.421189:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:2.0:1773941511.421192:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:2.0:1773941511.421196:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:2.0:1773941511.421198:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941511.421201:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941511.421225:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941511.421228:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941511.421231:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941511.421234:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941511.421239:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06506300. 00000020:00000040:2.0:1773941511.421242:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:2.0:1773941511.421246:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941511.421253:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941511.421256:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941511.421259:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941511.421262:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941511.421267:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d035a7400. 00000100:00000001:2.0:1773941511.421271:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941511.421277:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d06506300 x1860112291479680/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941511.421285:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941511.421288:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941511.421293:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000005464cf12] to pc [ptlrpcd_rcv+-1] req@ffff988d06506300 x1860112291479680/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941511.421314:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941511.421317:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:2.0:1773941511.421319:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941511.421321:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941511.421324:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941511.421326:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941511.421328:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6a80 going to sleep for 1 seconds 00000100:00000001:2.0:1773941511.421331:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941511.421334:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941511.421341:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941511.421346:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941511.421349:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941511.421350:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10835f80 x1860112291479552/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941527 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941511.421357:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06506300 x1860112291479680/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941511.421363:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941511.421370:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941511.421372:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941511.421374:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941511.421375:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941511.421380:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1773941511.421380:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1773941511.421383:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941511.421386:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941511.421392:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06506300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291479680:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000040:2.0:1773941511.421392:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10835f80 x1860112291479552/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941527 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941511.421398:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941511.421407:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:2.0:1773941511.421409:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10835f80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291479552:192.168.204.114@tcp:101:ll_cfg_requeue.0 02000000:00000001:3.0:1773941511.421410:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941511.421415:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1773941511.421419:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:3.0:1773941511.421421:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03523000. 02000000:00000001:3.0:1773941511.421423:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941511.421429:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941511.421433:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000010:3.0:1773941511.421434:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00000100:00000001:2.0:1773941511.421438:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000400:00000010:3.0:1773941511.421442:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00000100:00000001:2.0:1773941511.421442:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000200:3.0:1773941511.421447:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291479680, portal 25 00000100:00000040:2.0:1773941511.421451:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10835f80 x1860112291479552/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941527 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941511.421452:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941511.421454:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196583168 : -113743512968448 : ffff988d06506300) 00000100:00000040:3.0:1773941511.421463:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06506300 x1860112291479680/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941521 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941511.421471:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941511.421477:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000010:2.0:1773941511.421478:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb6a80. 00000100:00000001:2.0:1773941511.421483:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:2.0:1773941511.421485:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:3.0:1773941511.421491:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00000001:2.0:1773941511.421492:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000400:00000010:3.0:1773941511.421496:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac000. 00010000:00000001:2.0:1773941511.421497:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000200:3.0:1773941511.421504:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291479680, offset 0 00000020:00000001:2.0:1773941511.421506:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941511.421510:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030600 refcount=4 00000400:00000200:3.0:1773941511.421514:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000020:00000001:2.0:1773941511.421515:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090849792 : -113743618701824 : ffff988d00030600) 00010000:00000001:2.0:1773941511.421521:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090849792 : -113743618701824 : ffff988d00030600) 00000400:00000200:3.0:1773941511.421523:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:2.0:1773941511.421527:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e2913 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941511.421533:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941511.421540:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:2.0:1773941511.421544:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000200:3.0:1773941511.421548:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00010000:2.0:1773941511.421550:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e2913 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941511.421555:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:2.0:1773941511.421559:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000400:00000200:3.0:1773941511.421563:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00010000:2.0:1773941511.421563:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e2913 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941511.421573:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:3.0:1773941511.421574:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[24, 25] 00010000:00000001:2.0:1773941511.421575:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:2.0:1773941511.421579:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e2913 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941511.421585:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00010000:00000001:2.0:1773941511.421589:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:3.0:1773941511.421593:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [136:136:256:4294967295] 192.168.204.114@tcp LPNI seq info [26:26:8:4294967295] 00010000:00000001:2.0:1773941511.421598:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941511.421602:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:3.0:1773941511.421603:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:2.0:1773941511.421606:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e2913 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941511.421616:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00010000:00010000:2.0:1773941511.421620:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d00030600 already unused, calling callback (ffffffffc12b4c20) 10000000:00000001:2.0:1773941511.421625:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000800:00000200:3.0:1773941511.421626:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 10000000:00010000:2.0:1773941511.421629:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e2913 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000010:3.0:1773941511.421634:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74400. 00010000:00000001:2.0:1773941511.421635:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:2.0:1773941511.421638:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:2.0:1773941511.421639:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941511.421642:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030600 refcount=5 00000800:00000200:3.0:1773941511.421644:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000020:00000001:2.0:1773941511.421645:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090849792 : -113743618701824 : ffff988d00030600) 00010000:00000001:2.0:1773941511.421647:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090849792 : -113743618701824 : ffff988d00030600) 00010000:00000001:2.0:1773941511.421650:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:2.0:1773941511.421652:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e2913 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:2.0:1773941511.421660:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:2.0:1773941511.421661:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e2913 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941511.421667:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 10000000:01000000:2.0:1773941511.421669:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:2.0:1773941511.421674:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000800:00000200:3.0:1773941511.421675:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74400 type 1, nob 616 niov 1 nkiov 1 10000000:00000001:2.0:1773941511.421675:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941511.421680:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:2.0:1773941511.421682:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:2.0:1773941511.421685:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1773941511.421686:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1773941511.421687:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941511.421688:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941511.421690:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d00030600 with handle 0x8e86767622e2913 from hash 00000100:00000001:3.0:1773941511.421692:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941511.421699:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:2.0:1773941511.421699:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000100:00000001:3.0:1773941511.421702:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294903017 00010000:00000001:2.0:1773941511.421703:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941511.421706:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941511.421708:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1773941511.421710:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000100:00000001:3.0:1773941511.421711:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294903015 00010000:00000001:2.0:1773941511.421712:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:2.0:1773941511.421716:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941511.421718:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294903017 00010000:00000001:2.0:1773941511.421718:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941511.421721:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941511.421724:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294903015 10000000:00000001:2.0:1773941511.421724:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941511.421731:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00010000:2.0:1773941511.421731:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e2913 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941511.421737:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941511.421743:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941511.421743:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941511.421745:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941511.421747:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000100:00000001:3.0:1773941511.421748:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1773941511.421749:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:2.0:1773941511.421751:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941511.421752:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941511.421753:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1773941511.421755:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e2913 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941511.421756:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941511.421759:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1773941511.421762:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000040:2.0:1773941511.421763:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d21ec1400 count: 0 00000100:00000001:3.0:1773941511.421764:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941511.421767:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941511.421769:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941511.421777:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00000001:2.0:1773941511.421780:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000010:1.0:1773941511.421781:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74400. 00010000:00000001:2.0:1773941511.421784:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000200:1.0:1773941511.421787:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1773941511.421790:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000400:00000200:1.0:1773941511.421794:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000040:2.0:1773941511.421799:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10835f80 x1860112291479552/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941527 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1773941511.421800:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac000 00000400:00000010:1.0:1773941511.421803:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac000. 00000100:00000001:1.0:1773941511.421808:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941511.421815:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06506300 x1860112291479680/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941521 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941511.421816:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941511.421820:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:1.0:1773941511.421825:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000010:2.0:1773941511.421826:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d02ce7000. 00000100:00000040:1.0:1773941511.421829:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06506300 x1860112291479680/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941521 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1773941511.421833:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:1.0:1773941511.421837:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941511.421839:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:1.0:1773941511.421841:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000040:2.0:1773941511.421843:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941511.421847:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941511.421858:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065c5400. 02000000:00000001:2.0:1773941511.421864:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941511.421866:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941511.421869:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10835f80. 00000100:00000001:2.0:1773941511.421873:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941511.421875:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:2.0:1773941511.421878:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:2.0:1773941511.421881:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:2.0:1773941511.421885:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:2.0:1773941511.421890:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d054377c0. 00000020:00000010:2.0:1773941511.421896:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d08a4f400. 00000020:00000010:2.0:1773941511.421901:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d08a4fe00. 00000040:00000010:2.0:1773941511.421907:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d03356200. 00000020:00000010:2.0:1773941511.421912:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941511.421918:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d0381e000. 80000000:00000010:2.0:1773941511.421924:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d08bbb420. 00040000:00000010:2.0:1773941511.421930:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d0381f200. 20000000:00000010:2.0:1773941511.421936:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d0381e800. 10000000:00000040:2.0:1773941511.421941:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:2.0:1773941511.421944:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:2.0:1773941511.421949:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d08a4fe00. 00000040:00000010:2.0:1773941511.421956:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d03356200. 00000020:00000010:2.0:1773941511.422012:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941511.422019:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d0381e000. 80000000:00000010:2.0:1773941511.422023:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d08bbb420. 00040000:00000010:2.0:1773941511.422027:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d0381f200. 20000000:00000010:2.0:1773941511.422032:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d0381e800. 00000020:00000010:2.0:1773941511.422036:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d08a4f400. 10000000:00000010:2.0:1773941511.422040:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d054377c0. 10000000:01000000:2.0:1773941511.422044:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:2.0:1773941511.422049:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:2.0:1773941511.422051:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 00000800:00000001:0.0:1773941511.423341:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941511.423348:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941511.423356:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941511.423359:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941511.423363:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941511.423366:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941511.423371:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941511.423373:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941511.423376:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941511.423379:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324604080 00000400:00000200:0.0:1773941511.423383:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941511.423386:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xbc1 [1] + 0 00000800:00000001:0.0:1773941511.423390:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941511.423400:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941511.423402:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941511.423405:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941511.423408:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941511.423412:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06506300 x1860112291479680/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941521 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941511.423420:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d06506300 x1860112291479680/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941521 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941511.423432:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941511.423435:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941511.423438:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941511.423458:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941511.423460:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941511.423464:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941511.423465:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941511.423467:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941511.423471:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941511.423472:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941511.423474:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941511.423479:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06506300 x1860112291479680/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941521 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941511.423489:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06506300 x1860112291479680/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941521 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941511.423497:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941511.423499:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941511.423501:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941511.423511:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941511.423513:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941511.423515:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941511.423518:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941511.423522:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=12 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941511.423526:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941511.423529:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941511.423531:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941511.423534:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941511.423537:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941511.423539:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941511.423543:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06506300 x1860112291479680/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941521 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941511.423549:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941511.423550:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941511.423552:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941511.423554:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941511.423559:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941511.423560:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941511.423562:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941511.423563:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941511.423565:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941511.423567:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941511.423569:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941511.423571:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941511 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941511.423573:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941511.423579:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941511.423581:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941511.423583:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941511.423584:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941511.423585:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941511.423588:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941511.423590:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941511.423592:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941511.423600:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941511.423603:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941511.423605:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941511.423847:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941511.423852:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941511.423858:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06506300 x1860112291479680/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941521 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941511.423867:0:14724:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:3.0:1773941511.423868:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06506300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291479680:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941511.423877:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1773941511.423878:0:14724:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00000001:3.0:1773941511.423880:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00100000:1.0:1773941511.423883:0:14724:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000040:3.0:1773941511.423886:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06506300 x1860112291479680/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941521 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941511.423887:0:14724:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941511.423889:0:14724:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:1.0:1773941511.423894:0:14724:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941511.423898:0:14724:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:3.0:1773941511.423899:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:1.0:1773941511.423902:0:14724:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941511.423903:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941511.423907:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03523000. 00000001:00000001:1.0:1773941511.423907:0:14724:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:3.0:1773941511.423912:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941511.423915:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:1.0:1773941511.423916:0:14724:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000040:3.0:1773941511.423917:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000001:00000001:1.0:1773941511.423919:0:14724:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941511.423921:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 10000000:00000001:1.0:1773941511.423922:0:14724:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941511.423924:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d035a7400. 00000100:00000001:1.0:1773941511.423925:0:14724:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000001:3.0:1773941511.423928:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941511.423931:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:1.0:1773941511.423931:0:14724:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000010:3.0:1773941511.423934:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06506300. 00000100:00000001:1.0:1773941511.423936:0:14724:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:3.0:1773941511.423939:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941511.423940:0:14724:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941511.423942:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941511.423943:0:14724:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941511.423946:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1773941511.423948:0:14724:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:3.0:1773941511.423951:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941511.423952:0:14724:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:3.0:1773941511.423954:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 10000000:01000000:1.0:1773941511.423955:0:14724:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000100:00000001:3.0:1773941511.424001:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941511.424003:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:00000040:1.0:1773941511.424014:0:14724:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:1.0:1773941511.424030:0:14724:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941511.424032:0:14724:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941511.424035:0:14724:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941511.424041:0:14724:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:1.0:1773941511.424045:0:14724:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941511.424048:0:14724:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941511.424050:0:14724:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941511.424053:0:14724:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941511.424055:0:14724:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941511.428869:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:2.0:1773941511.428876:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:2.0:1773941511.428900:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941511.428906:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00010000:00000010:2.1:1773941511.431019:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d21ec1400. 00010000:00000010:2.1:1773941511.431035:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d00030600. 00000020:00000001:2.0:1773941512.187165:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941512.187174:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941512.187181:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1773941512.699086:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294903017, at 4294903040 00000100:00000040:1.0:1773941512.699096:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294903017, at 4294903040 00000100:00000010:0.0:1773941512.699100:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580b40. 00000100:00000010:1.0:1773941512.699115:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d065803c0. 10000000:00000001:2.0:1773941512.955096:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:2.0:1773941512.955104:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941512.955109:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941512.955112:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941512.955114:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941512.955115:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:2.0:1773941512.955119:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:2.0:1773941512.955120:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:2.0:1773941512.955123:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:2.0:1773941512.955126:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:2.0:1773941512.955128:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:2.0:1773941512.955137:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06506a00. 00000020:00000040:2.0:1773941512.955142:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941512.955149:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941512.955154:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941512.955160:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941512.955169:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d0381e800. 00000100:00000001:2.0:1773941512.955177:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941512.955183:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:2.0:1773941512.955186:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:2.0:1773941512.955194:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d071ee100. 00010000:00000001:2.0:1773941512.955201:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:2.0:1773941512.955206:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d089dfa00. 00000020:00000001:2.0:1773941512.955211:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:2.0:1773941512.955213:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d089dfa00 with handle 0x8e86767622e291a to hash 00000020:00000001:2.0:1773941512.955217:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:2.0:1773941512.955219:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941512.955223:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941512.955228:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941512.955232:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941512.955235:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e291a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941512.955261:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e291a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941512.955275:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e291a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941512.955283:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:2.0:1773941512.955286:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:2.0:1773941512.955304:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d09350540. 00000100:00000001:2.0:1773941512.955311:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330245121344 : -113743464430272 : ffff988d09350540) 00000100:00000001:2.0:1773941512.955316:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941512.955319:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584960 : -113743512966656 : ffff988d06506a00) 00000100:00000001:2.0:1773941512.955326:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941512.955333:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941512.955336:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:2.0:1773941512.955340:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941512.955356:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06506a00 x1860112291479808/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941512.955364:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941512.955367:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941512.955371:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941512.955374:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941512.955376:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941512.955380:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291479808:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941512.955385:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941512.955391:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941512.955393:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941512.955396:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941512.955400:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a7c00. 02000000:00000001:2.0:1773941512.955403:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941512.955411:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941512.955420:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000400:00000010:2.0:1773941512.955428:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000200:2.0:1773941512.955436:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291479808, portal 25 00000100:00000001:2.0:1773941512.955437:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941512.955439:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584960 : -113743512966656 : ffff988d06506a00) 00000100:00000040:2.0:1773941512.955444:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06506a00 x1860112291479808/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941528 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941512.955450:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941512.955453:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:2.0:1773941512.955455:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516e58. 00000100:00000200:2.0:1773941512.955458:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291479808, offset 0 00000400:00000200:2.0:1773941512.955464:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941512.955472:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941512.955479:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941512.955484:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941512.955489:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941512.955493:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941512.955495:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941512.955497:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[26, 25] 00000400:00000200:2.0:1773941512.955500:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:2.0:1773941512.955503:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [137:137:256:4294967295] 192.168.204.124@tcp LPNI seq info [27:27:8:4294967295] 00000400:00000200:2.0:1773941512.955507:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:2.0:1773941512.955518:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:2.0:1773941512.955524:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:2.0:1773941512.955528:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0db900. 00000800:00000200:2.0:1773941512.955534:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:2.0:1773941512.955541:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:2.0:1773941512.955546:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0db900 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:2.0:1773941512.955559:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941512.955563:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941512.955566:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294904551 00000100:00000010:2.0:1773941512.955574:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d08bbb420. 00000100:00000001:2.0:1773941512.955581:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1773941512.955587:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294904551 00000100:00000010:2.0:1773941512.955591:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d08bbba20. 00000100:00000001:2.0:1773941512.955593:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1773941512.955596:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941512.955598:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941512.955600:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941512.955603:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:2.0:1773941512.955605:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d09350540 going to sleep for 11 seconds 00000100:00000001:2.0:1773941512.955608:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941512.955612:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941512.955647:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941512.955652:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0db900. 00000400:00000200:1.0:1773941512.955657:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941512.955663:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941512.955667:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516e58 00000400:00000010:1.0:1773941512.955670:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516e58. 00000100:00000001:1.0:1773941512.955674:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941512.955678:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06506a00 x1860112291479808/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941528 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941512.955688:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941512.955691:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d06506a00 x1860112291479808/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941528 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941512.955702:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941512.955704:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941512.956696:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941512.956708:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941512.956711:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941512.956713:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941512.956721:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941512.956727:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941512.956734:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941512.956738:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941512.956752:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941512.956755:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324604100 00000400:00000200:0.0:1773941512.956760:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941512.956764:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xbc9 [1] + 224 00000800:00000001:0.0:1773941512.956771:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941512.956779:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941512.956781:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941512.956785:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941512.956790:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941512.956797:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06506a00 x1860112291479808/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941528 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941512.956811:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d06506a00 x1860112291479808/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941528 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941512.956826:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941512.956833:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941512.956835:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941512.956856:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941512.956862:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941512.956863:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941512.956865:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941512.956869:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941512.956870:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941512.956872:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941512.956877:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06506a00 x1860112291479808/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941528 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941512.956885:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06506a00 x1860112291479808/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941528 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941512.956891:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941512.956892:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941512.956894:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941512.956896:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941512.956898:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941512.956900:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941512.956902:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941512.956908:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=13 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941512.956910:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941512.956913:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:2.0:1773941512.956916:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:2.0:1773941512.956924:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941512.956927:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:2.0:1773941512.956929:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1773941512.956933:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1773941512.956936:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1773941512.956939:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941512.956941:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941512.956942:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941512.956944:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:2.0:1773941512.956986:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d06506a00 x1860112291479808/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941528 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941512.956994:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941512.956995:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941512.956997:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941512.957000:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941512.957002:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:2.0:1773941512.957004:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941512.957005:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941512.957006:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941512.957007:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941512.957009:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941512.957010:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941512.957012:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1773941512.957014:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1773941512.957016:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941512.957019:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 234 00000400:00000200:2.0:1773941512.957023:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941512.957026:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:2.0:1773941512.957029:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941512.957032:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941512.957034:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941512.957036:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:2.0:1773941512.957043:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941512.957045:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941512.957047:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941512.957049:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941512.957051:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941512.957053:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:2.0:1773941512.957055:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941512.957057:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941512.957058:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941512.957059:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:2.0:1773941512.957062:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:2.0:1773941512.957064:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:2.0:1773941512.957066:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941512.957068:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941512.957070:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941512.957072:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941512.957075:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941512.957076:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941512.957079:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06507b80. 00000020:00000040:2.0:1773941512.957080:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:2.0:1773941512.957083:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941512.957084:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941512.957085:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941512.957087:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941512.957088:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941512.957092:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d035a5000. 00000100:00000001:2.0:1773941512.957095:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941512.957099:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d06507b80 x1860112291479936/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941512.957105:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941512.957107:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941512.957111:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000df552709] to pc [ptlrpcd_rcv+-1] req@ffff988d06507b80 x1860112291479936/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941512.957130:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941512.957132:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:2.0:1773941512.957133:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941512.957136:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941512.957137:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941512.957139:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941512.957140:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d09350540 going to sleep for 1 seconds 00000100:00000001:2.0:1773941512.957142:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941512.957144:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941512.957148:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06506a00 x1860112291479808/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941528 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941512.957154:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941512.957157:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941512.957159:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941512.957160:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941512.957164:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941512.957165:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1773941512.957169:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06506a00 x1860112291479808/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941528 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941512.957170:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00100000:2.0:1773941512.957176:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291479808:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941512.957182:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941512.957185:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:3.0:1773941512.957186:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06507b80 x1860112291479936/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941512.957187:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:2.0:1773941512.957190:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941512.957195:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06506a00 x1860112291479808/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941528 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941512.957204:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941512.957207:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941512.957208:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941512.957211:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000010:2.0:1773941512.957212:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d09350540. 02000000:00000001:3.0:1773941512.957216:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941512.957217:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:2.0:1773941512.957219:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:3.0:1773941512.957221:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941512.957223:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00100000:3.0:1773941512.957227:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06507b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291479936:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941512.957230:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941512.957234:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00000001:2.0:1773941512.957235:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 02000000:00000001:3.0:1773941512.957236:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941512.957238:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:3.0:1773941512.957240:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:2.0:1773941512.957241:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089dfa00 refcount=4 00000020:00000001:2.0:1773941512.957245:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941512.957247:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 02000000:00000010:3.0:1773941512.957248:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03523000. 00010000:00010000:2.0:1773941512.957251:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e291a lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941512.957254:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941512.957263:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000200:3.0:1773941512.957264:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:2.0:1773941512.957271:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e291a lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:3.0:1773941512.957272:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00000400:00000010:3.0:1773941512.957280:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00010000:00000001:2.0:1773941512.957285:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000200:3.0:1773941512.957287:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291479936, portal 25 00010000:00010000:2.0:1773941512.957290:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e291a lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941512.957292:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941512.957295:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196589440 : -113743512962176 : ffff988d06507b80) 00000100:00000040:3.0:1773941512.957303:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06507b80 x1860112291479936/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941522 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941512.957322:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:2.0:1773941512.957323:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000040:3.0:1773941512.957326:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00000001:2.0:1773941512.957327:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000010:3.0:1773941512.957330:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac7f8. 00010000:00010000:2.0:1773941512.957333:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e291a lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:3.0:1773941512.957334:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291479936, offset 0 00000400:00000200:3.0:1773941512.957344:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00000001:2.0:1773941512.957347:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941512.957350:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941512.957351:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:2.0:1773941512.957354:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:3.0:1773941512.957357:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00010000:2.0:1773941512.957358:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e291a lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941512.957364:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941512.957370:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00010000:2.0:1773941512.957371:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d089dfa00 already unused, calling callback (ffffffffc12b4c20) 00000400:00000200:3.0:1773941512.957373:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 10000000:00000001:2.0:1773941512.957377:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:3.0:1773941512.957378:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 10000000:00010000:2.0:1773941512.957380:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e291a lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941512.957387:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000400:00000200:3.0:1773941512.957389:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[26, 27] 00000400:00000200:3.0:1773941512.957393:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00010000:00000001:2.0:1773941512.957397:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:2.0:1773941512.957399:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:3.0:1773941512.957401:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [138:138:256:4294967295] 192.168.204.114@tcp LPNI seq info [28:28:8:4294967295] 00000020:00000040:2.0:1773941512.957402:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089dfa00 refcount=5 00000020:00000001:2.0:1773941512.957407:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941512.957411:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00000400:00000200:3.0:1773941512.957412:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:2.0:1773941512.957418:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:2.0:1773941512.957422:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e291a lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941512.957425:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 10000000:00000001:2.0:1773941512.957434:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000800:00000200:3.0:1773941512.957436:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 10000000:00010000:2.0:1773941512.957437:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e291a lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000010:3.0:1773941512.957441:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74400. 10000000:01000000:2.0:1773941512.957444:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:2.0:1773941512.957447:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:2.0:1773941512.957450:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941512.957451:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00010000:00010000:2.0:1773941512.957453:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:2.0:1773941512.957455:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:2.0:1773941512.957458:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000800:00000200:3.0:1773941512.957461:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00010000:00000001:2.0:1773941512.957462:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1773941512.957464:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000800:00000200:3.0:1773941512.957466:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74400 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:2.0:1773941512.957467:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941512.957473:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d089dfa00 with handle 0x8e86767622e291a from hash 00010000:00000001:2.0:1773941512.957475:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1773941512.957476:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941512.957477:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941512.957478:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1773941512.957479:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:2.0:1773941512.957480:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:2.0:1773941512.957482:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941512.957483:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941512.957484:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941512.957486:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941512.957488:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941512.957490:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:2.0:1773941512.957492:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e291a lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941512.957493:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294904553 00000100:00000001:3.0:1773941512.957503:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294904551 00010000:00000001:2.0:1773941512.957510:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941512.957511:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294904553 00010000:00000001:2.0:1773941512.957512:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941512.957516:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000100:00000001:3.0:1773941512.957518:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294904551 00010000:00000001:2.0:1773941512.957518:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:2.0:1773941512.957521:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941512.957523:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941512.957524:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941512.957525:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00010000:2.0:1773941512.957527:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e291a lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941512.957528:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941512.957534:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1773941512.957537:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d071ee100 count: 0 00000100:00000001:3.0:1773941512.957538:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941512.957547:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941512.957549:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00010000:00000001:2.0:1773941512.957550:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:3.0:1773941512.957551:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:2.0:1773941512.957553:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000800:00000200:1.0:1773941512.957554:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1773941512.957555:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941512.957556:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1773941512.957559:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941512.957562:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1773941512.957563:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74400. 00000100:00000040:2.0:1773941512.957565:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06506a00 x1860112291479808/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941528 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1773941512.957570:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1773941512.957573:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941512.957577:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941512.957581:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a7c00. 00000400:00000200:1.0:1773941512.957581:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 02000000:00000001:2.0:1773941512.957587:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000400:00000200:1.0:1773941512.957589:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac7f8 00000020:00000001:2.0:1773941512.957590:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000400:00000010:1.0:1773941512.957592:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac7f8. 00000020:00000040:2.0:1773941512.957593:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941512.957597:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941512.957598:0:2393:0:(events.c:34:request_out_callback()) Process entered 02000000:00000010:2.0:1773941512.957600:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d0381e800. 00000100:00000200:1.0:1773941512.957605:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06507b80 x1860112291479936/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941522 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1773941512.957606:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941512.957609:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941512.957612:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06506a00. 00000100:00000001:2.0:1773941512.957617:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941512.957621:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941512.957621:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 10000000:00000001:2.0:1773941512.957624:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:1.0:1773941512.957627:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06507b80 x1860112291479936/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941522 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:01000000:2.0:1773941512.957629:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:2.0:1773941512.957635:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:2.0:1773941512.957640:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d054377c0. 00000020:00000010:2.0:1773941512.957644:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d0381e800. 00000100:00000001:1.0:1773941512.957647:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941512.957648:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:2.0:1773941512.957657:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d0381f200. 00000040:00000010:2.0:1773941512.957663:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d0381e000. 00000020:00000010:2.0:1773941512.957669:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941512.957675:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d03356200. 80000000:00000010:2.0:1773941512.957679:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d08bbbd80. 00040000:00000010:2.0:1773941512.957685:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d08a4f400. 20000000:00000010:2.0:1773941512.957691:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d08a4fe00. 10000000:00000040:2.0:1773941512.957697:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:2.0:1773941512.957699:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:2.0:1773941512.957704:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d0381f200. 00000040:00000010:2.0:1773941512.957709:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d0381e000. 00000020:00000010:2.0:1773941512.957712:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941512.957715:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d03356200. 80000000:00000010:2.0:1773941512.957718:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d08bbbd80. 00040000:00000010:2.0:1773941512.957721:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d08a4f400. 20000000:00000010:2.0:1773941512.957724:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d08a4fe00. 00000020:00000010:2.0:1773941512.957726:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0381e800. 10000000:00000010:2.0:1773941512.957728:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d054377c0. 10000000:01000000:2.0:1773941512.957731:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:2.0:1773941512.957733:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:2.0:1773941512.957735:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:2.0:1773941512.957737:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:2.0:1773941512.957738:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:2.0:1773941512.957739:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941512.957741:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941512.958862:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941512.958872:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941512.958876:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941512.958879:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941512.958885:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941512.958891:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941512.958898:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941512.958903:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941512.958909:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941512.958913:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324604180 00000400:00000200:0.0:1773941512.958918:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941512.958922:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xbd1 [1] + 0 00000800:00000001:0.0:1773941512.958928:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941512.958936:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941512.958938:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941512.958941:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941512.958984:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941512.958989:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06507b80 x1860112291479936/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941522 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941512.958998:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d06507b80 x1860112291479936/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941522 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941512.959012:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941512.959016:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941512.959019:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941512.959053:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941512.959057:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941512.959061:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941512.959063:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941512.959065:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941512.959071:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941512.959080:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941512.959084:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941512.959097:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06507b80 x1860112291479936/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941522 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941512.959110:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06507b80 x1860112291479936/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941522 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941512.959119:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941512.959121:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941512.959123:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941512.959126:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941512.959127:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941512.959129:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941512.959132:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941512.959136:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=13 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941512.959139:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941512.959142:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941512.959144:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941512.959147:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941512.959150:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941512.959154:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941512.959159:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06507b80 x1860112291479936/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941522 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941512.959167:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941512.959170:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941512.959172:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941512.959175:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941512.959180:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941512.959182:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941512.959184:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941512.959186:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941512.959188:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941512.959191:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941512.959194:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941512.959198:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941512 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941512.959208:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941512.959216:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941512.959219:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941512.959221:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941512.959223:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941512.959225:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941512.959229:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941512.959232:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941512.959235:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941512.959239:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941512.959243:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941512.959246:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941512.959450:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941512.959454:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941512.959459:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06507b80 x1860112291479936/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941522 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941512.959467:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06507b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291479936:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941512.959469:0:14726:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941512.959472:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941512.959480:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941512.959483:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06507b80 x1860112291479936/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941522 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1773941512.959488:0:14726:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00000001:2.0:1773941512.959492:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00100000:3.0:1773941512.959495:0:14726:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000001:2.0:1773941512.959496:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941512.959499:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03523000. 00000100:00000001:3.0:1773941512.959501:0:14726:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:2.0:1773941512.959503:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 10000000:01000000:3.0:1773941512.959506:0:14726:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000020:00000001:2.0:1773941512.959506:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941512.959510:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000001:00000001:3.0:1773941512.959513:0:14726:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:2.0:1773941512.959513:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941512.959517:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d035a5000. 00000001:00000001:3.0:1773941512.959519:0:14726:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941512.959520:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941512.959523:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941512.959525:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06507b80. 00000001:00000001:3.0:1773941512.959528:0:14726:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941512.959528:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941512.959531:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:3.0:1773941512.959534:0:14726:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:2.0:1773941512.959534:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:3.0:1773941512.959539:0:14726:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941512.959540:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000001:00000001:3.0:1773941512.959543:0:14726:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941512.959543:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941512.959550:0:14726:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941512.959550:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941512.959553:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941512.959556:0:14726:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941512.959561:0:14726:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:3.0:1773941512.959564:0:14726:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:3.0:1773941512.959567:0:14726:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941512.959570:0:14726:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1773941512.959573:0:14726:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:3.0:1773941512.959576:0:14726:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941512.959578:0:14726:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:3.0:1773941512.959580:0:14726:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:3.0:1773941512.959600:0:14726:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:3.0:1773941512.959603:0:14726:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941512.959611:0:14726:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:3.0:1773941512.959616:0:14726:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:3.0:1773941512.959620:0:14726:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941512.959623:0:14726:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941512.959625:0:14726:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941512.959628:0:14726:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941512.959630:0:14726:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:2.1:1773941512.964004:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d071ee100. 00010000:00000010:2.1:1773941512.964014:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d089dfa00. 00000020:00000001:2.0:1773941513.211170:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941513.211178:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941513.211183:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941514.235089:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294904553, at 4294904576 00000100:00000010:1.0:1773941514.235110:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d08bbba20. 00000100:00000040:0.0:1773941514.235116:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294904553, at 4294904576 00000100:00000010:0.0:1773941514.235132:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d08bbb420. 00000020:00000001:2.0:1773941514.235148:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941514.235153:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941514.235157:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:2.0:1773941514.747138:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:2.0:1773941514.747147:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941514.747152:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941514.747154:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941514.747156:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941514.747157:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:2.0:1773941514.747160:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:2.0:1773941514.747161:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:2.0:1773941514.747164:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:2.0:1773941514.747166:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:2.0:1773941514.747191:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:2.0:1773941514.747199:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06507b80. 00000020:00000040:2.0:1773941514.747204:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941514.747210:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941514.747213:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941514.747219:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941514.747228:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d08a4fe00. 00000100:00000001:2.0:1773941514.747236:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941514.747240:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:2.0:1773941514.747244:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:2.0:1773941514.747252:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d071ee100. 00010000:00000001:2.0:1773941514.747258:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:2.0:1773941514.747261:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d089dfa00. 00000020:00000001:2.0:1773941514.747266:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:2.0:1773941514.747269:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d089dfa00 with handle 0x8e86767622e2921 to hash 00000020:00000001:2.0:1773941514.747270:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:2.0:1773941514.747272:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941514.747276:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941514.747281:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941514.747285:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941514.747290:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2921 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941514.747303:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2921 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941514.747318:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2921 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941514.747325:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:2.0:1773941514.747327:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:2.0:1773941514.747336:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d09350540. 00000100:00000001:2.0:1773941514.747338:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330245121344 : -113743464430272 : ffff988d09350540) 00000100:00000001:2.0:1773941514.747342:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941514.747349:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196589440 : -113743512962176 : ffff988d06507b80) 00000100:00000001:2.0:1773941514.747355:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941514.747360:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941514.747363:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:2.0:1773941514.747365:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941514.747378:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06507b80 x1860112291480064/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941514.747388:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941514.747391:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941514.747396:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941514.747399:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941514.747402:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941514.747406:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06507b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291480064:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941514.747410:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941514.747416:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941514.747419:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941514.747422:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941514.747428:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a5000. 02000000:00000001:2.0:1773941514.747430:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941514.747439:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941514.747444:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000400:00000010:2.0:1773941514.747452:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000200:2.0:1773941514.747461:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291480064, portal 25 00000100:00000001:2.0:1773941514.747463:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941514.747465:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196589440 : -113743512962176 : ffff988d06507b80) 00000100:00000040:2.0:1773941514.747470:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06507b80 x1860112291480064/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941530 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941514.747479:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941514.747482:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:2.0:1773941514.747485:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516110. 00000100:00000200:2.0:1773941514.747489:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291480064, offset 0 00000400:00000200:2.0:1773941514.747497:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941514.747508:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941514.747526:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941514.747534:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941514.747539:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941514.747544:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941514.747547:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941514.747550:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[28, 27] 00000400:00000200:2.0:1773941514.747554:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:2.0:1773941514.747559:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [139:139:256:4294967295] 192.168.204.124@tcp LPNI seq info [29:29:8:4294967295] 00000400:00000200:2.0:1773941514.747564:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:2.0:1773941514.747577:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:2.0:1773941514.747585:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:2.0:1773941514.747591:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0db000. 00000800:00000200:2.0:1773941514.747599:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:2.0:1773941514.747609:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:2.0:1773941514.747615:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0db000 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:2.0:1773941514.747637:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941514.747642:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941514.747645:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294906343 00000100:00000010:2.0:1773941514.747650:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d08bbbd80. 00000100:00000001:2.0:1773941514.747659:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1773941514.747667:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294906343 00000100:00000010:2.0:1773941514.747672:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d08bbbe40. 00000100:00000001:2.0:1773941514.747675:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1773941514.747680:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941514.747683:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941514.747686:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941514.747688:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:2.0:1773941514.747692:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d09350540 going to sleep for 11 seconds 00000100:00000001:2.0:1773941514.747696:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941514.747699:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941514.747798:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941514.747804:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0db000. 00000400:00000200:1.0:1773941514.747812:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941514.747819:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941514.747826:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516110 00000400:00000010:1.0:1773941514.747830:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516110. 00000100:00000001:1.0:1773941514.747835:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941514.747841:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06507b80 x1860112291480064/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941530 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941514.747853:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941514.747857:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d06507b80 x1860112291480064/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941530 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941514.747866:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941514.747876:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941514.748803:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941514.748816:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941514.748819:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941514.748822:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941514.748842:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941514.748848:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941514.748856:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941514.748861:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941514.748867:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941514.748873:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324604200 00000400:00000200:0.0:1773941514.748878:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941514.748885:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xbd9 [1] + 224 00000800:00000001:0.0:1773941514.748892:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941514.748902:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941514.748904:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941514.748908:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941514.748925:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941514.748932:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06507b80 x1860112291480064/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941530 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941514.748945:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d06507b80 x1860112291480064/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941530 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941514.749014:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941514.749022:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941514.749028:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941514.749037:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941514.749048:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941514.749050:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941514.749053:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941514.749056:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941514.749059:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941514.749062:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941514.749067:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06507b80 x1860112291480064/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941530 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941514.749076:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06507b80 x1860112291480064/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941530 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941514.749082:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941514.749084:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941514.749086:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941514.749088:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941514.749090:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941514.749092:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941514.749094:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941514.749100:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=15 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941514.749102:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941514.749105:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:2.0:1773941514.749108:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:2.0:1773941514.749110:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941514.749113:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:2.0:1773941514.749115:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1773941514.749120:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1773941514.749123:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1773941514.749126:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941514.749128:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941514.749129:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941514.749130:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:2.0:1773941514.749133:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d06507b80 x1860112291480064/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941530 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941514.749145:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941514.749147:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941514.749148:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941514.749151:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941514.749153:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:2.0:1773941514.749154:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941514.749156:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941514.749157:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941514.749158:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941514.749159:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941514.749160:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941514.749162:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1773941514.749163:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1773941514.749166:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941514.749168:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 236 00000400:00000200:2.0:1773941514.749171:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941514.749174:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:2.0:1773941514.749177:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941514.749179:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941514.749181:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941514.749183:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:2.0:1773941514.749185:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941514.749187:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941514.749189:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941514.749192:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941514.749193:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941514.749195:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:2.0:1773941514.749197:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941514.749198:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941514.749200:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941514.749201:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:2.0:1773941514.749203:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:2.0:1773941514.749208:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:2.0:1773941514.749210:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941514.749213:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941514.749225:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941514.749227:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941514.749229:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941514.749232:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941514.749236:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06506a00. 00000020:00000040:2.0:1773941514.749239:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:2.0:1773941514.749242:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941514.749244:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941514.749247:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941514.749249:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941514.749251:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941514.749256:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d035a7c00. 00000100:00000001:2.0:1773941514.749260:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941514.749267:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d06506a00 x1860112291480192/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941514.749276:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941514.749279:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941514.749284:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6b603d8] to pc [ptlrpcd_rcv+-1] req@ffff988d06506a00 x1860112291480192/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941514.749309:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941514.749311:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:2.0:1773941514.749313:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941514.749315:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941514.749317:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941514.749318:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941514.749320:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d09350540 going to sleep for 1 seconds 00000100:00000001:2.0:1773941514.749321:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941514.749323:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941514.749327:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06507b80 x1860112291480064/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941530 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941514.749341:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941514.749344:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941514.749346:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941514.749348:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941514.749349:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1773941514.749354:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06507b80 x1860112291480064/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941530 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941514.749356:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941514.749360:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00100000:2.0:1773941514.749365:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06507b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291480064:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941514.749370:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941514.749372:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:3.0:1773941514.749373:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06506a00 x1860112291480192/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941514.749374:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:2.0:1773941514.749377:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941514.749381:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06507b80 x1860112291480064/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941530 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941514.749403:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941514.749403:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941514.749407:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941514.749408:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d09350540. 00000100:00000001:2.0:1773941514.749412:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 02000000:00000001:3.0:1773941514.749414:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1773941514.749415:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:3.0:1773941514.749419:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00010000:00000001:2.0:1773941514.749419:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:2.0:1773941514.749424:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 02000000:00000001:3.0:1773941514.749425:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941514.749426:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941514.749429:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089dfa00 refcount=4 00000020:00000001:2.0:1773941514.749432:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00000100:00100000:3.0:1773941514.749435:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291480192:192.168.204.114@tcp:250:ll_cfg_requeue.0 00010000:00000001:2.0:1773941514.749435:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00010000:2.0:1773941514.749441:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2921 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941514.749446:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941514.749450:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00000001:2.0:1773941514.749451:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00010000:00010000:2.0:1773941514.749456:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2921 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941514.749465:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941514.749465:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:2.0:1773941514.749469:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2921 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941514.749470:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:2.0:1773941514.749477:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 02000000:00000010:3.0:1773941514.749478:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03520c00. 00010000:00000001:2.0:1773941514.749479:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:2.0:1773941514.749482:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2921 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941514.749484:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941514.749491:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941514.749493:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941514.749495:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:2.0:1773941514.749496:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:2.0:1773941514.749501:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2921 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:3.0:1773941514.749503:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00010000:00010000:2.0:1773941514.749511:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d089dfa00 already unused, calling callback (ffffffffc12b4c20) 00000400:00000010:3.0:1773941514.749514:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 10000000:00000001:2.0:1773941514.749514:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:2.0:1773941514.749517:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2921 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:3.0:1773941514.749521:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291480192, portal 25 00010000:00000001:2.0:1773941514.749522:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:2.0:1773941514.749525:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000001:3.0:1773941514.749528:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000020:00000001:2.0:1773941514.749528:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941514.749531:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089dfa00 refcount=5 00000100:00000001:3.0:1773941514.749533:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584960 : -113743512966656 : ffff988d06506a00) 00000020:00000001:2.0:1773941514.749533:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941514.749539:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941514.749542:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:2.0:1773941514.749548:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2921 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:3.0:1773941514.749549:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06506a00 x1860112291480192/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941524 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000001:2.0:1773941514.749560:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000001:3.0:1773941514.749567:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 10000000:00010000:2.0:1773941514.749568:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2921 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1773941514.749572:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:2.0:1773941514.749574:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:2.0:1773941514.749576:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941514.749578:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:2.0:1773941514.749581:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000100:00000040:3.0:1773941514.749585:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00000001:2.0:1773941514.749586:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1773941514.749589:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1773941514.749591:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000010:3.0:1773941514.749592:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac660. 00010000:00000001:2.0:1773941514.749593:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941514.749597:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d089dfa00 with handle 0x8e86767622e2921 from hash 00000100:00000200:3.0:1773941514.749600:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291480192, offset 0 00010000:00000001:2.0:1773941514.749600:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1773941514.749602:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941514.749603:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941514.749606:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1773941514.749608:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:2.0:1773941514.749609:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000400:00000200:3.0:1773941514.749614:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00000001:2.0:1773941514.749615:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941514.749617:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941514.749620:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941514.749621:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941514.749623:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2921 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941514.749625:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:2.0:1773941514.749632:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941514.749634:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:3.0:1773941514.749635:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:2.0:1773941514.749636:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:2.0:1773941514.749639:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:2.0:1773941514.749642:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:3.0:1773941514.749644:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:2.0:1773941514.749644:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941514.749647:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1773941514.749650:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2921 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941514.749652:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941514.749659:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000040:2.0:1773941514.749659:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d071ee100 count: 0 00000400:00000200:3.0:1773941514.749663:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941514.749672:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[28, 29] 00010000:00000001:2.0:1773941514.749677:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:3.0:1773941514.749678:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00010000:00000001:2.0:1773941514.749680:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941514.749685:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000400:00000200:3.0:1773941514.749686:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [140:140:256:4294967295] 192.168.204.114@tcp LPNI seq info [30:30:8:4294967295] 00000100:00000040:2.0:1773941514.749690:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06507b80 x1860112291480064/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941530 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941514.749699:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941514.749702:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941514.749705:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941514.749708:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a5000. 02000000:00000001:2.0:1773941514.749713:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941514.749716:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000400:00000200:3.0:1773941514.749718:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000020:00000040:2.0:1773941514.749718:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941514.749723:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941514.749727:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d08a4fe00. 00000800:00000200:3.0:1773941514.749731:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 02000000:00000001:2.0:1773941514.749732:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941514.749734:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941514.749737:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06507b80. 00000100:00000001:2.0:1773941514.749740:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000800:00000010:3.0:1773941514.749741:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74500. 00000100:00000001:2.0:1773941514.749742:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1773941514.749750:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 10000000:00000001:2.0:1773941514.749753:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:2.0:1773941514.749756:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:2.0:1773941514.749759:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000800:00000200:3.0:1773941514.749763:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 10000000:00000010:2.0:1773941514.749767:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d054377c0. 00000800:00000200:3.0:1773941514.749769:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74500 type 1, nob 616 niov 1 nkiov 1 00000020:00000010:2.0:1773941514.749772:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d08a4fe00. 00000020:00000010:2.0:1773941514.749781:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d08a4f400. 00000040:00000010:2.0:1773941514.749786:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d03356200. 00000100:00000001:3.0:1773941514.749788:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1773941514.749792:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d09305000. 00000400:00000200:3.0:1773941514.749796:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000020:00000010:2.0:1773941514.749799:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d0381e800. 00000100:00000001:3.0:1773941514.749800:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294906345 80000000:00000010:2.0:1773941514.749805:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d08bbbcc0. 00000100:00000001:3.0:1773941514.749810:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294906343 00040000:00000010:2.0:1773941514.749812:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d0381e000. 20000000:00000010:2.0:1773941514.749819:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d0381f200. 00000100:00000001:3.0:1773941514.749820:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294906345 00000100:00000001:3.0:1773941514.749827:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294906343 00000100:00000001:3.0:1773941514.749831:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 10000000:00000040:2.0:1773941514.749831:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 00000100:00000001:3.0:1773941514.749835:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941514.749840:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941514.749841:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941514.749848:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:2.0:1773941514.749851:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d08a4f400. 00000040:00000010:2.0:1773941514.749858:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d03356200. 00000100:00000001:3.0:1773941514.749862:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941514.749864:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1773941514.749870:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000010:2.0:1773941514.749873:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d09305000. 00000800:00000200:1.0:1773941514.749873:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1773941514.749874:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941514.749877:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1773941514.749880:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74500. 00000100:00000001:3.0:1773941514.749882:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1773941514.749883:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d0381e800. 00000400:00000200:1.0:1773941514.749885:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 80000000:00000010:2.0:1773941514.749891:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d08bbbcc0. 00000400:00000200:1.0:1773941514.749894:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00040000:00000010:2.0:1773941514.749897:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d0381e000. 00000400:00000200:1.0:1773941514.749899:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac660 20000000:00000010:2.0:1773941514.749902:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d0381f200. 00000400:00000010:1.0:1773941514.749902:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac660. 00000020:00000010:2.0:1773941514.749907:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d08a4fe00. 00000100:00000001:1.0:1773941514.749908:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941514.749925:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06506a00 x1860112291480192/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941524 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000010:2.0:1773941514.749928:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d054377c0. 10000000:01000000:2.0:1773941514.749933:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 00000100:00000001:1.0:1773941514.749938:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 10000000:00000001:2.0:1773941514.749939:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:2.0:1773941514.749943:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 00000100:00000040:1.0:1773941514.749945:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06506a00 x1860112291480192/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941524 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000001:2.0:1773941514.749946:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:2.0:1773941514.749947:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:2.0:1773941514.749949:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941514.749951:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000100:00000001:1.0:1773941514.750004:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941514.750007:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941514.751104:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941514.751111:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941514.751114:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941514.751116:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941514.751121:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941514.751125:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941514.751131:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941514.751135:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941514.751139:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941514.751142:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324604280 00000400:00000200:0.0:1773941514.751147:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941514.751151:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xbe1 [1] + 0 00000800:00000001:0.0:1773941514.751156:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941514.751163:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941514.751166:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941514.751171:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941514.751175:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941514.751180:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06506a00 x1860112291480192/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941524 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941514.751191:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d06506a00 x1860112291480192/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941524 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941514.751207:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941514.751212:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941514.751215:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941514.751242:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941514.751246:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941514.751250:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941514.751251:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941514.751254:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941514.751259:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941514.751261:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941514.751263:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941514.751269:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06506a00 x1860112291480192/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941524 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941514.751280:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06506a00 x1860112291480192/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941524 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941514.751287:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941514.751289:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941514.751291:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941514.751293:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941514.751294:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941514.751296:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941514.751299:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941514.751302:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=15 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941514.751306:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941514.751308:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941514.751311:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941514.751313:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941514.751316:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941514.751318:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941514.751323:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06506a00 x1860112291480192/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941524 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941514.751330:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941514.751332:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941514.751340:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941514.751342:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941514.751347:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941514.751350:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941514.751351:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941514.751354:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941514.751356:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941514.751358:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941514.751361:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941514.751365:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941514 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941514.751368:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941514.751374:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941514.751376:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941514.751378:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941514.751380:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941514.751382:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941514.751386:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941514.751388:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941514.751390:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941514.751394:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941514.751397:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941514.751400:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941514.751661:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941514.751665:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941514.751670:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06506a00 x1860112291480192/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941524 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941514.751678:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291480192:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941514.751681:0:14727:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941514.751682:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941514.751685:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941514.751688:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06506a00 x1860112291480192/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941524 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941514.751693:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00080000:3.0:1773941514.751694:0:14727:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 02000000:00000001:2.0:1773941514.751695:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941514.751699:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03520c00. 00000100:00100000:3.0:1773941514.751701:0:14727:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000001:2.0:1773941514.751702:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941514.751705:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:3.0:1773941514.751707:0:14727:0:(obd_class.h:1215:obd_import_event()) Process entered 00000020:00000040:2.0:1773941514.751708:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 10000000:01000000:3.0:1773941514.751711:0:14727:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000020:00000001:2.0:1773941514.751712:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941514.751716:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d035a7c00. 00000001:00000001:3.0:1773941514.751718:0:14727:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:2.0:1773941514.751719:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000001:00000001:3.0:1773941514.751722:0:14727:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941514.751726:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000001:00000001:3.0:1773941514.751728:0:14727:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941514.751728:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06506a00. 00000100:00000001:2.0:1773941514.751733:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000001:00000001:3.0:1773941514.751734:0:14727:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:2.0:1773941514.751736:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:3.0:1773941514.751737:0:14727:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941514.751739:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:3.0:1773941514.751742:0:14727:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941514.751743:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 10000000:00000001:3.0:1773941514.751748:0:14727:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941514.751751:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941514.751753:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941514.751754:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941514.751761:0:14727:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941514.751766:0:14727:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:3.0:1773941514.751769:0:14727:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:3.0:1773941514.751772:0:14727:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941514.751774:0:14727:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1773941514.751777:0:14727:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:3.0:1773941514.751780:0:14727:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941514.751782:0:14727:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:3.0:1773941514.751784:0:14727:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:3.0:1773941514.751800:0:14727:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:3.0:1773941514.751803:0:14727:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941514.751805:0:14727:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:3.0:1773941514.751811:0:14727:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:3.0:1773941514.751816:0:14727:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941514.751820:0:14727:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941514.751822:0:14727:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941514.751825:0:14727:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941514.751826:0:14727:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:2.1:1773941514.754994:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d071ee100. 00010000:00000010:2.1:1773941514.755002:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d089dfa00. 00000020:00000001:2.0:1773941515.259178:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941515.259189:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941515.259195:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1773941515.515115:0:461:0:(pinger.c:229:ptlrpc_pinger_process_import()) c1c1ff21-5f68-4bf6-bdc4-258644b24465->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1773941515.515131:0:461:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1773941515.515145:0:461:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10835f80. 00000020:00000040:0.0:1773941515.515154:0:461:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941515.515180:0:461:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941515.515184:0:461:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941515.515191:0:461:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941515.515202:0:461:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff988d0a20bd00. 00000100:00000001:0.0:1773941515.515210:0:461:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941515.515225:0:461:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging c1c1ff21-5f68-4bf6-bdc4-258644b24465->MGS req@ffff988d10835f80 x1860112291480320/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1773941515.515239:0:461:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941515.515244:0:461:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941515.515253:0:461:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000082649650] to pc [ptlrpcd_00_01+1] req@ffff988d10835f80 x1860112291480320/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941515.515288:0:461:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941515.515295:0:461:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (243) 00000100:00000001:1.0:1773941515.515307:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941515.515308:0:2401:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941515.515315:0:2402:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941515.515315:0:2401:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941515.515319:0:2402:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941515.515327:0:2402:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10835f80 x1860112291480320/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941515.515336:0:2402:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941515.515338:0:2402:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941515.515342:0:2402:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941515.515345:0:2402:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941515.515347:0:2402:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941515.515350:0:2402:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10835f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2402:1860112291480320:192.168.204.114@tcp:400:kworker.0 00000100:00000001:1.0:1773941515.515353:0:2402:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941515.515360:0:2402:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941515.515362:0:2402:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941515.515364:0:2402:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941515.515368:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff988d181fc200. 02000000:00000001:1.0:1773941515.515370:0:2402:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941515.515378:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941515.515389:0:2402:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c5d8. 00000400:00000010:1.0:1773941515.515396:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000100:00000200:1.0:1773941515.515403:0:2402:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112291480320, portal 25 00000100:00000001:1.0:1773941515.515405:0:2402:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941515.515407:0:2402:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367696768 : -113743341854848 : ffff988d10835f80) 00000100:00000040:1.0:1773941515.515412:0:2402:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10835f80 x1860112291480320/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941531 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941515.515419:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941515.515421:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941515.515423:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113dec38. 00000100:00000200:1.0:1773941515.515425:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1860112291480320, offset 0 00000400:00000200:1.0:1773941515.515431:0:2402:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941515.515439:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941515.515447:0:2402:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941515.515451:0:2402:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941515.515455:0:2402:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941515.515459:0:2402:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941515.515461:0:2402:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941515.515464:0:2402:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[30, 29] 00000400:00000200:1.0:1773941515.515467:0:2402:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:1.0:1773941515.515470:0:2402:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [141:141:256:4294967295] 192.168.204.124@tcp LPNI seq info [31:31:8:4294967295] 00000400:00000200:1.0:1773941515.515474:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941515.515482:0:2402:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:1.0:1773941515.515488:0:2402:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:1.0:1773941515.515492:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfd800. 00000800:00000200:1.0:1773941515.515498:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:1.0:1773941515.515506:0:2402:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:1.0:1773941515.515510:0:2402:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfd800 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1773941515.515525:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941515.515529:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941515.515532:0:2402:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294907111 00000100:00000001:1.0:1773941515.515544:0:2402:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294906345 00000100:00000001:1.0:1773941515.515548:0:2402:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294907111 00000100:00000001:1.0:1773941515.515552:0:2402:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294906345 00000100:00000001:1.0:1773941515.515555:0:2402:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941515.515557:0:2402:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941515.515561:0:2402:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941515.515563:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941515.515570:0:2402:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941515.515572:0:2402:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:1.0:1773941515.515574:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941515.515575:0:2402:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941515.515576:0:2402:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941515.515578:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941515.515674:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941515.515679:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfd800. 00000400:00000200:0.0:1773941515.515686:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941515.515693:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941515.515699:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113dec38 00000400:00000010:0.0:1773941515.515703:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113dec38. 00000100:00000001:0.0:1773941515.515707:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941515.515713:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10835f80 x1860112291480320/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941531 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941515.515724:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941515.515728:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10835f80 x1860112291480320/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941531 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941515.515736:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941515.515738:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941515.516778:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941515.516792:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941515.516796:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941515.516799:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941515.516816:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941515.516820:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941515.516834:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941515.516838:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941515.516842:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941515.516845:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324604300 00000400:00000200:0.0:1773941515.516849:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941515.516854:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xbe9 [1] + 224 00000800:00000001:0.0:1773941515.516861:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941515.516868:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941515.516871:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941515.516873:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941515.516877:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941515.516881:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10835f80 x1860112291480320/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941531 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941515.516890:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff988d10835f80 x1860112291480320/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941531 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941515.516917:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941515.516924:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941515.516927:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941515.516950:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941515.516953:0:2402:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941515.516958:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941515.516974:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941515.516977:0:2402:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:1.0:1773941515.516980:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:1.0:1773941515.516982:0:2402:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:1.0:1773941515.516984:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941515.516988:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10835f80 x1860112291480320/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941531 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941515.516997:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10835f80 x1860112291480320/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941531 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941515.517002:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941515.517004:0:2402:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941515.517006:0:2402:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941515.517009:0:2402:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941515.517011:0:2402:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941515.517012:0:2402:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941515.517015:0:2402:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941515.517021:0:2402:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=16 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941515.517024:0:2402:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941515.517026:0:2402:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1773941515.517033:0:2402:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941515.517036:0:2402:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941515.517039:0:2402:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941515.517043:0:2402:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941515.517047:0:2402:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941515.517050:0:2402:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941515.517052:0:2402:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941515.517054:0:2402:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941515.517055:0:2402:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941515.517058:0:2402:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d10835f80 x1860112291480320/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941531 ref 1 fl Rpc:RNQU/200/0 rc 0/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941515.517063:0:2402:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941515.517064:0:2402:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941515.517066:0:2402:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941515.517070:0:2402:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941515.517073:0:2402:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941515.517074:0:2402:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941515.517076:0:2402:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941515.517077:0:2402:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941515.517079:0:2402:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941515.517080:0:2402:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941515.517082:0:2402:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941515.517085:0:2402:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941515.517087:0:2402:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941515.517091:0:2402:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941515.517094:0:2402:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 237 00000400:00000200:1.0:1773941515.517100:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941515.517104:0:2402:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941515.517120:0:2402:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941515.517128:0:2402:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941515.517131:0:2402:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941515.517134:0:2402:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941515.517138:0:2402:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941515.517147:0:2402:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941515.517151:0:2402:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941515.517154:0:2402:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941515.517156:0:2402:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941515.517158:0:2402:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941515.517161:0:2402:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941515.517163:0:2402:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941515.517164:0:2402:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941515.517166:0:2402:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941515.517168:0:2402:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941515.517171:0:2402:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941515.517173:0:2402:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941515.517175:0:2402:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941515.517177:0:2402:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941515.517179:0:2402:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941515.517181:0:2402:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941515.517183:0:2402:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941515.517187:0:2402:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c2300. 00000020:00000040:1.0:1773941515.517189:0:2402:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941515.517192:0:2402:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941515.517193:0:2402:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941515.517195:0:2402:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941515.517197:0:2402:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941515.517199:0:2402:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941515.517203:0:2402:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d11188800. 00000100:00000001:1.0:1773941515.517206:0:2402:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941515.517210:0:2402:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c2300 x1860112291480448/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941515.517217:0:2402:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941515.517219:0:2402:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941515.517222:0:2402:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093c3ab5c] to pc [ptlrpcd_rcv+-1] req@ffff988d203c2300 x1860112291480448/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941515.517254:0:2402:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941515.517257:0:2402:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941515.517259:0:2402:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:1.0:1773941515.517266:0:2402:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10835f80 x1860112291480320/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941531 ref 1 fl Rpc:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941515.517274:0:2402:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941515.517278:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941515.517280:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941515.517285:0:2402:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10835f80 x1860112291480320/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941531 ref 1 fl Interpret:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941515.517294:0:2402:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10835f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2402:1860112291480320:192.168.204.114@tcp:400:kworker.0 00000100:00000001:1.0:1773941515.517299:0:2402:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941515.517301:0:2402:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941515.517303:0:2402:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10835f80 x1860112291480320/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941531 ref 1 fl Complete:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941515.517307:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941515.517309:0:2402:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1773941515.517312:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1773941515.517313:0:2402:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1773941515.517317:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 02000000:00000010:1.0:1773941515.517318:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff988d181fc200. 02000000:00000001:1.0:1773941515.517323:0:2402:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941515.517325:0:2402:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941515.517329:0:2402:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000040:2.0:1773941515.517330:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c2300 x1860112291480448/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1773941515.517331:0:2402:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941515.517336:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff988d0a20bd00. 02000000:00000001:1.0:1773941515.517341:0:2402:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941515.517343:0:2402:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941515.517346:0:2402:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10835f80. 00000100:00000001:2.0:1773941515.517350:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941515.517351:0:2402:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941515.517354:0:2402:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941515.517355:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941515.517356:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941515.517361:0:2402:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:2.0:1773941515.517362:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941515.517365:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941515.517369:0:2402:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941515.517372:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941515.517372:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941515.517375:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941515.517380:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291480448:192.168.204.114@tcp:250:ptlrpcd_00_01.0 00000100:00000001:2.0:1773941515.517386:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941515.517389:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941515.517391:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941515.517393:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941515.517398:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a7c00. 02000000:00000001:2.0:1773941515.517401:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941515.517406:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941515.517410:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000400:00000010:2.0:1773941515.517415:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000200:2.0:1773941515.517418:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291480448, portal 25 00000100:00000001:2.0:1773941515.517421:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941515.517423:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631463680 : -113743078087936 : ffff988d203c2300) 00000100:00000040:2.0:1773941515.517434:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c2300 x1860112291480448/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941525 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941515.517440:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941515.517442:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:2.0:1773941515.517444:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516bb0. 00000100:00000200:2.0:1773941515.517447:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291480448, offset 0 00000400:00000200:2.0:1773941515.517451:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941515.517455:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941515.517458:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941515.517460:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941515.517462:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941515.517464:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941515.517466:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941515.517468:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[30, 31] 00000400:00000200:2.0:1773941515.517471:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:2.0:1773941515.517473:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [142:142:256:4294967295] 192.168.204.114@tcp LPNI seq info [32:32:8:4294967295] 00000400:00000200:2.0:1773941515.517476:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941515.517481:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:2.0:1773941515.517485:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:2.0:1773941515.517488:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0dbd00. 00000800:00000200:2.0:1773941515.517491:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:2.0:1773941515.517497:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:2.0:1773941515.517499:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0dbd00 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:2.0:1773941515.517510:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941515.517512:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941515.517514:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294907113 00000100:00000001:2.0:1773941515.517517:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294907111 00000100:00000001:2.0:1773941515.517520:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294907113 00000100:00000001:2.0:1773941515.517522:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294907111 00000100:00000001:2.0:1773941515.517524:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941515.517530:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941515.517532:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941515.517534:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941515.517537:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941515.517539:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941515.517541:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941515.517542:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941515.517544:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941515.517545:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941515.517599:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941515.517603:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0dbd00. 00000400:00000200:0.0:1773941515.517609:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941515.517614:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941515.517617:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516bb0 00000400:00000010:0.0:1773941515.517620:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516bb0. 00000100:00000001:0.0:1773941515.517623:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941515.517628:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c2300 x1860112291480448/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941525 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941515.517638:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941515.517642:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c2300 x1860112291480448/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941525 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941515.517662:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941515.517664:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941515.519138:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941515.519147:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941515.519150:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941515.519152:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941515.519157:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941515.519162:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941515.519168:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941515.519171:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941515.519175:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941515.519178:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324604380 00000400:00000200:0.0:1773941515.519183:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941515.519188:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xbf9 [1] + 0 00000800:00000001:0.0:1773941515.519193:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941515.519199:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941515.519201:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941515.519205:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941515.519210:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941515.519236:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c2300 x1860112291480448/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941525 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941515.519249:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c2300 x1860112291480448/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941525 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941515.519264:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941515.519269:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941515.519273:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941515.519306:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941515.519309:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941515.519313:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941515.519315:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941515.519317:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941515.519322:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941515.519324:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941515.519327:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941515.519333:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c2300 x1860112291480448/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941525 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941515.519345:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c2300 x1860112291480448/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941525 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941515.519354:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941515.519356:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941515.519358:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941515.519360:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941515.519362:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941515.519364:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941515.519366:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941515.519370:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=16 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941515.519374:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941515.519376:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941515.519378:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941515.519382:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941515.519384:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941515.519388:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941515.519392:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c2300 x1860112291480448/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941525 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941515.519414:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941515.519416:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941515.519418:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941515.519421:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941515.519428:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941515.519431:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941515.519432:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941515.519434:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941515.519437:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941515.519439:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941515.519442:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:2.0:1773941515.519447:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941515 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941515.519450:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941515.519457:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:2.0:1773941515.519460:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941515.519462:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941515.519464:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941515.519466:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941515.519471:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:2.0:1773941515.519474:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941515.519477:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941515.519481:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941515.519484:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:2.0:1773941515.519488:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941515.519773:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941515.519782:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941515.519798:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c2300 x1860112291480448/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941525 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941515.519809:0:14728:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:3.0:1773941515.519811:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291480448:192.168.204.114@tcp:250:ptlrpcd_00_01.0 00000100:00000001:3.0:1773941515.519822:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941515.519828:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941515.519832:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c2300 x1860112291480448/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941525 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1773941515.519837:0:14728:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00000001:3.0:1773941515.519847:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00100000:2.0:1773941515.519850:0:14728:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000001:3.0:1773941515.519853:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1773941515.519855:0:14728:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941515.519861:0:14728:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000010:3.0:1773941515.519862:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a7c00. 00000001:00000001:2.0:1773941515.519871:0:14728:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:3.0:1773941515.519872:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941515.519877:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:2.0:1773941515.519878:0:14728:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000040:3.0:1773941515.519881:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000001:00000001:2.0:1773941515.519892:0:14728:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:2.0:1773941515.519895:0:14728:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941515.519912:0:14728:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000001:3.0:1773941515.519914:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:2.0:1773941515.519918:0:14728:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941515.519919:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d11188800. 02000000:00000001:3.0:1773941515.519931:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 10000000:00000001:2.0:1773941515.519931:0:14728:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941515.519936:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941515.519940:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c2300. 00000100:00000001:2.0:1773941515.519940:0:14728:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941515.519952:0:14728:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:3.0:1773941515.519953:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941515.520007:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941515.520012:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941515.520020:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941515.520021:0:14728:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:3.0:1773941515.520027:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941515.520028:0:14728:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:2.0:1773941515.520035:0:14728:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941515.520036:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941515.520040:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1773941515.520042:0:14728:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941515.520048:0:14728:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941515.520050:0:14728:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:2.0:1773941515.520052:0:14728:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:2.0:1773941515.520070:0:14728:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:2.0:1773941515.520072:0:14728:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941515.520074:0:14728:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:2.0:1773941515.520080:0:14728:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:2.0:1773941515.520084:0:14728:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941515.520087:0:14728:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941515.520089:0:14728:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941515.520092:0:14728:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941515.520094:0:14728:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941516.219110:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1773941516.219119:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941516.219126:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941516.219129:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941516.219131:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941516.219132:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1773941516.219152:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:0.0:1773941516.219154:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:0.0:1773941516.219157:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:0.0:1773941516.219160:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1773941516.219162:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:0.0:1773941516.219170:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10836680. 00000020:00000040:0.0:1773941516.219175:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941516.219180:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941516.219184:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941516.219189:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941516.219197:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065c5a00. 00000100:00000001:0.0:1773941516.219205:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941516.219210:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1773941516.219213:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1773941516.219221:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d21ec1400. 00010000:00000001:0.0:1773941516.219227:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1773941516.219230:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d00031800. 00000020:00000001:0.0:1773941516.219235:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:0.0:1773941516.219237:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d00031800 with handle 0x8e86767622e2928 to hash 00000020:00000001:0.0:1773941516.219239:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:0.0:1773941516.219241:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330090854400 : -113743618697216 : ffff988d00031800) 00010000:00000001:0.0:1773941516.219244:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330090854400 : -113743618697216 : ffff988d00031800) 00010000:00000001:0.0:1773941516.219249:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941516.219253:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941516.219258:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00031800/0x8e86767622e2928 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941516.219269:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d00031800/0x8e86767622e2928 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941516.219282:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d00031800/0x8e86767622e2928 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941516.219289:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1773941516.219308:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941516.219318:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb66c0. 00000100:00000001:0.0:1773941516.219321:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330139633344 : -113743569918272 : ffff988d02eb66c0) 00000100:00000001:0.0:1773941516.219324:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941516.219325:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367698560 : -113743341853056 : ffff988d10836680) 00000100:00000001:0.0:1773941516.219330:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941516.219334:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941516.219337:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941516.219339:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941516.219350:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10836680 x1860112291480576/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941516.219361:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941516.219363:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941516.219367:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941516.219370:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941516.219372:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941516.219376:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10836680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291480576:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941516.219381:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941516.219387:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941516.219390:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941516.219393:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941516.219397:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d02ce5800. 02000000:00000001:0.0:1773941516.219400:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941516.219407:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941516.219411:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0014b5d8. 00000400:00000010:0.0:1773941516.219418:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833770. 00000100:00000200:0.0:1773941516.219425:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291480576, portal 25 00000100:00000001:0.0:1773941516.219427:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941516.219429:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367698560 : -113743341853056 : ffff988d10836680) 00000100:00000040:0.0:1773941516.219434:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10836680 x1860112291480576/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941532 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941516.219442:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941516.219445:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941516.219453:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833a18. 00000100:00000200:0.0:1773941516.219457:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291480576, offset 0 00000400:00000200:0.0:1773941516.219464:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941516.219473:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941516.219480:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941516.219485:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941516.219490:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941516.219495:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941516.219498:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941516.219502:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[32, 31] 00000400:00000200:0.0:1773941516.219505:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:0.0:1773941516.219510:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [143:143:256:4294967295] 192.168.204.124@tcp LPNI seq info [33:33:8:4294967295] 00000400:00000200:0.0:1773941516.219514:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941516.219524:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:0.0:1773941516.219531:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:0.0:1773941516.219536:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a20bd00. 00000800:00000200:0.0:1773941516.219543:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:0.0:1773941516.219551:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:0.0:1773941516.219556:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a20bd00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1773941516.219572:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941516.219576:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941516.219579:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294907815 00000100:00000001:0.0:1773941516.219586:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294907113 00000100:00000001:0.0:1773941516.219592:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294907815 00000100:00000001:0.0:1773941516.219596:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294907113 00000100:00000001:0.0:1773941516.219600:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941516.219604:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941516.219607:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941516.219608:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1773941516.219611:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb66c0 going to sleep for 11 seconds 00000100:00000001:0.0:1773941516.219620:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941516.219623:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941516.219716:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941516.219722:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a20bd00. 00000400:00000200:1.0:1773941516.219727:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941516.219733:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941516.219739:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833a18 00000400:00000010:1.0:1773941516.219741:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833a18. 00000100:00000001:1.0:1773941516.219746:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941516.219751:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10836680 x1860112291480576/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941532 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941516.219761:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941516.219765:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d10836680 x1860112291480576/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941532 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941516.219772:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941516.219774:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941516.220818:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941516.220833:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941516.220837:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941516.220839:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941516.220844:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941516.220849:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941516.220856:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941516.220860:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941516.220864:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941516.220869:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324604400 00000400:00000200:0.0:1773941516.220874:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941516.220879:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xc01 [1] + 224 00000800:00000001:0.0:1773941516.220901:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941516.220928:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941516.220931:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941516.220935:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941516.220940:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941516.220945:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10836680 x1860112291480576/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941532 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941516.220956:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d10836680 x1860112291480576/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941532 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941516.220996:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941516.221004:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941516.221010:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941516.221044:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941516.221050:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941516.221053:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941516.221055:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0014b5d8. 00000400:00000200:2.0:1773941516.221060:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833770 00000400:00000010:2.0:1773941516.221061:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833770. 00000100:00000001:2.0:1773941516.221064:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941516.221070:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10836680 x1860112291480576/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941532 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941516.221079:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10836680 x1860112291480576/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941532 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941516.221084:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941516.221093:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941516.221095:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941516.221097:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941516.221099:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941516.221100:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941516.221102:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941516.221108:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=17 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941516.221111:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941516.221113:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:2.0:1773941516.221116:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:2.0:1773941516.221118:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941516.221120:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:2.0:1773941516.221123:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1773941516.221126:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1773941516.221130:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1773941516.221132:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941516.221134:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941516.221136:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941516.221137:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:2.0:1773941516.221140:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d10836680 x1860112291480576/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941532 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941516.221145:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941516.221146:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941516.221147:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941516.221149:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941516.221151:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:2.0:1773941516.221152:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941516.221154:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941516.221154:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941516.221155:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941516.221156:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941516.221157:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941516.221158:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1773941516.221160:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1773941516.221165:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941516.221167:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 238 00000400:00000200:2.0:1773941516.221171:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941516.221174:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:2.0:1773941516.221177:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941516.221180:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941516.221182:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941516.221184:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:2.0:1773941516.221186:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941516.221188:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941516.221189:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941516.221191:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941516.221193:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941516.221194:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:2.0:1773941516.221196:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941516.221198:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941516.221199:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941516.221200:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:2.0:1773941516.221202:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:2.0:1773941516.221204:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:2.0:1773941516.221206:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941516.221207:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941516.221210:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941516.221211:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941516.221213:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941516.221214:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941516.221217:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06506a00. 00000020:00000040:2.0:1773941516.221219:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:2.0:1773941516.221221:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941516.221222:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941516.221224:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941516.221225:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941516.221230:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941516.221234:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d035a5000. 00000100:00000001:2.0:1773941516.221237:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941516.221241:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d06506a00 x1860112291480704/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941516.221247:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941516.221249:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941516.221253:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6b603d8] to pc [ptlrpcd_rcv+-1] req@ffff988d06506a00 x1860112291480704/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941516.221270:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941516.221272:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:2.0:1773941516.221273:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941516.221275:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941516.221277:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941516.221278:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941516.221280:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb66c0 going to sleep for 1 seconds 00000100:00000001:2.0:1773941516.221281:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941516.221283:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941516.221286:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10836680 x1860112291480576/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941532 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941516.221292:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941516.221294:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941516.221297:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941516.221299:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941516.221302:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10836680 x1860112291480576/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941532 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941516.221307:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:2.0:1773941516.221309:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10836680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291480576:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941516.221316:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941516.221317:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941516.221323:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1773941516.221325:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941516.221328:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:2.0:1773941516.221333:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941516.221337:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06506a00 x1860112291480704/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941516.221343:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10836680 x1860112291480576/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941532 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941516.221395:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941516.221396:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941516.221398:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941516.221400:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb66c0. 02000000:00000001:3.0:1773941516.221404:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1773941516.221406:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 02000000:00000001:3.0:1773941516.221408:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941516.221408:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:3.0:1773941516.221421:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941516.221422:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:2.0:1773941516.221431:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00100000:3.0:1773941516.221436:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291480704:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000020:00000001:2.0:1773941516.221437:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941516.221443:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00031800 refcount=4 00000100:00000001:3.0:1773941516.221445:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000001:2.0:1773941516.221449:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090854400 : -113743618697216 : ffff988d00031800) 02000000:00000001:3.0:1773941516.221452:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00000001:2.0:1773941516.221453:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090854400 : -113743618697216 : ffff988d00031800) 02000000:00000001:3.0:1773941516.221456:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941516.221458:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d00031800/0x8e86767622e2928 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941516.221460:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941516.221469:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03521c00. 00010000:00000001:2.0:1773941516.221470:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 02000000:00000001:3.0:1773941516.221476:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941516.221479:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d00031800/0x8e86767622e2928 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941516.221493:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941516.221503:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00010000:00000001:2.0:1773941516.221503:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:2.0:1773941516.221511:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00031800/0x8e86767622e2928 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:3.0:1773941516.221514:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00010000:00000001:2.0:1773941516.221525:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941516.221526:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:2.0:1773941516.221528:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d00031800/0x8e86767622e2928 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941516.221533:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000200:3.0:1773941516.221534:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291480704, portal 25 00010000:00000001:2.0:1773941516.221538:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941516.221539:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941516.221544:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584960 : -113743512966656 : ffff988d06506a00) 00010000:00000001:2.0:1773941516.221551:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:2.0:1773941516.221563:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d00031800/0x8e86767622e2928 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:3.0:1773941516.221567:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06506a00 x1860112291480704/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941526 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941516.221601:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00010000:2.0:1773941516.221604:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d00031800 already unused, calling callback (ffffffffc12b4c20) 00000100:00000040:3.0:1773941516.221605:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:3.0:1773941516.221609:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acdd0. 10000000:00000001:2.0:1773941516.221611:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000200:3.0:1773941516.221614:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291480704, offset 0 10000000:00010000:2.0:1773941516.221617:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d00031800/0x8e86767622e2928 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941516.221623:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941516.221634:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:2.0:1773941516.221636:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000400:00000200:3.0:1773941516.221641:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:2.0:1773941516.221641:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:2.0:1773941516.221648:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:3.0:1773941516.221651:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000040:2.0:1773941516.221654:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00031800 refcount=5 00000400:00000200:3.0:1773941516.221657:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941516.221662:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000020:00000001:2.0:1773941516.221663:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090854400 : -113743618697216 : ffff988d00031800) 00000400:00000200:3.0:1773941516.221667:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:2.0:1773941516.221670:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090854400 : -113743618697216 : ffff988d00031800) 00000400:00000200:3.0:1773941516.221678:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[32, 33] 00010000:00000001:2.0:1773941516.221681:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:3.0:1773941516.221685:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00010000:00010000:2.0:1773941516.221686:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d00031800/0x8e86767622e2928 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941516.221697:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [144:144:256:4294967295] 192.168.204.114@tcp LPNI seq info [34:34:8:4294967295] 10000000:00000001:2.0:1773941516.221714:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:3.0:1773941516.221716:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 10000000:00010000:2.0:1773941516.221719:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d00031800/0x8e86767622e2928 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1773941516.221725:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:2.0:1773941516.221730:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:2.0:1773941516.221734:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941516.221740:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000400:00000200:3.0:1773941516.221745:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00010000:00000001:2.0:1773941516.221745:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:2.0:1773941516.221752:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1773941516.221755:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1773941516.221758:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941516.221759:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941516.221766:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d00031800 with handle 0x8e86767622e2928 from hash 00000800:00000200:3.0:1773941516.221767:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00010000:00000001:2.0:1773941516.221773:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1773941516.221775:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000010:3.0:1773941516.221778:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74f00. 00010000:00000001:2.0:1773941516.221784:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941516.221786:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1773941516.221787:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:2.0:1773941516.221789:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000800:00000200:3.0:1773941516.221796:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00010000:00000001:2.0:1773941516.221796:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941516.221801:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941516.221804:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941516.221806:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941516.221814:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00010000:00010000:2.0:1773941516.221816:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d00031800/0x8e86767622e2928 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941516.221819:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74f00 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:2.0:1773941516.221835:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941516.221836:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941516.221837:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:2.0:1773941516.221838:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:2.0:1773941516.221839:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941516.221840:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941516.221841:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1773941516.221842:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d00031800/0x8e86767622e2928 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941516.221844:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1773941516.221857:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d21ec1400 count: 0 00000400:00000200:3.0:1773941516.221858:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:3.0:1773941516.221867:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294907817 00000100:00000001:3.0:1773941516.221877:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294907815 00010000:00000001:2.0:1773941516.221880:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941516.221896:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941516.221900:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294907817 00000100:00000001:2.0:1773941516.221901:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1773941516.221911:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294907815 00000100:00000040:2.0:1773941516.221913:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10836680 x1860112291480576/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941532 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941516.221918:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941516.221923:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941516.221926:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1773941516.221931:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941516.221932:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d02ce5800. 00000100:00000001:3.0:1773941516.221941:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941516.221941:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941516.221947:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:3.0:1773941516.221948:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1773941516.221952:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000800:00000200:1.0:1773941516.221952:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1773941516.221957:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:2.0:1773941516.221957:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 00000800:00000010:1.0:1773941516.221959:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74f00. 00000100:00000001:3.0:1773941516.221991:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1773941516.221994:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941516.221996:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941516.221998:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941516.222004:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1773941516.222005:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941516.222008:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065c5a00. 02000000:00000001:2.0:1773941516.222022:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000400:00000200:1.0:1773941516.222023:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941516.222028:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acdd0 00000400:00000010:1.0:1773941516.222033:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acdd0. 02000000:00000001:2.0:1773941516.222034:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1773941516.222039:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000010:2.0:1773941516.222040:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10836680. 00000100:00000001:2.0:1773941516.222047:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000200:1.0:1773941516.222047:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06506a00 x1860112291480704/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941526 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941516.222051:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:2.0:1773941516.222058:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941516.222060:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 10000000:01000000:2.0:1773941516.222064:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000100:00000040:1.0:1773941516.222065:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06506a00 x1860112291480704/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941526 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000001:2.0:1773941516.222070:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000100:00000001:1.0:1773941516.222073:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941516.222075:0:2393:0:(events.c:68:request_out_callback()) Process leaving 10000000:00000010:2.0:1773941516.222077:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d05437280. 00000020:00000010:2.0:1773941516.222084:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d0381f200. 00000020:00000010:2.0:1773941516.222090:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d0381e000. 00000040:00000010:2.0:1773941516.222095:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d0381e800. 00000020:00000010:2.0:1773941516.222101:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941516.222108:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d03356200. 80000000:00000010:2.0:1773941516.222113:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d08bbbcc0. 00040000:00000010:2.0:1773941516.222118:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d08a4fe00. 20000000:00000010:2.0:1773941516.222121:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d08a4f400. 10000000:00000040:2.0:1773941516.222125:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:2.0:1773941516.222127:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:2.0:1773941516.222132:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d0381e000. 00000040:00000010:2.0:1773941516.222137:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d0381e800. 00000020:00000010:2.0:1773941516.222142:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941516.222146:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d03356200. 80000000:00000010:2.0:1773941516.222151:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d08bbbcc0. 00040000:00000010:2.0:1773941516.222155:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d08a4fe00. 20000000:00000010:2.0:1773941516.222159:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d08a4f400. 00000020:00000010:2.0:1773941516.222161:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0381f200. 10000000:00000010:2.0:1773941516.222164:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d05437280. 10000000:01000000:2.0:1773941516.222168:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:2.0:1773941516.222172:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:2.0:1773941516.222174:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:2.0:1773941516.222176:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:2.0:1773941516.222177:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:2.0:1773941516.222178:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941516.222180:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941516.223459:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941516.223469:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941516.223472:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941516.223475:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941516.223480:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941516.223486:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941516.223493:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941516.223503:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941516.223508:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941516.223512:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324604480 00000400:00000200:0.0:1773941516.223516:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941516.223520:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xc11 [1] + 0 00000800:00000001:0.0:1773941516.223526:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941516.223532:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941516.223535:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941516.223539:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941516.223543:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941516.223548:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06506a00 x1860112291480704/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941526 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941516.223558:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d06506a00 x1860112291480704/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941526 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941516.223576:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941516.223581:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941516.223584:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941516.223605:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941516.223608:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941516.223612:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941516.223613:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941516.223616:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941516.223620:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941516.223621:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941516.223624:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941516.223629:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06506a00 x1860112291480704/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941526 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941516.223638:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06506a00 x1860112291480704/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941526 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941516.223643:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941516.223644:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941516.223646:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941516.223648:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941516.223650:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941516.223652:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941516.223654:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941516.223659:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=17 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941516.223662:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941516.223664:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941516.223665:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941516.223668:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941516.223670:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941516.223672:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941516.223675:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06506a00 x1860112291480704/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941526 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941516.223691:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941516.223693:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941516.223694:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941516.223696:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941516.223702:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941516.223704:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941516.223706:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941516.223708:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941516.223710:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941516.223712:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941516.223714:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941516.223718:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941516 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941516.223721:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941516.223729:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941516.223733:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941516.223736:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941516.223738:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941516.223740:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941516.223744:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941516.223747:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941516.223750:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941516.223754:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941516.223758:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941516.223761:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941516.224021:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941516.224027:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941516.224035:0:14729:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000040:2.0:1773941516.224041:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06506a00 x1860112291480704/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941526 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1773941516.224046:0:14729:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:3.0:1773941516.224056:0:14729:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00100000:2.0:1773941516.224070:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291480704:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941516.224084:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941516.224087:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941516.224094:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06506a00 x1860112291480704/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941526 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941516.224098:0:14729:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941516.224102:0:14729:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000001:2.0:1773941516.224106:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:3.0:1773941516.224108:0:14729:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:2.0:1773941516.224110:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:3.0:1773941516.224113:0:14729:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000010:2.0:1773941516.224114:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03521c00. 02000000:00000001:2.0:1773941516.224118:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941516.224126:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:3.0:1773941516.224130:0:14729:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941516.224130:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000001:00000001:3.0:1773941516.224142:0:14729:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:2.0:1773941516.224144:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:3.0:1773941516.224153:0:14729:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000010:2.0:1773941516.224154:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d035a5000. 02000000:00000001:2.0:1773941516.224162:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000001:00000001:3.0:1773941516.224165:0:14729:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941516.224165:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 10000000:00000001:3.0:1773941516.224172:0:14729:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941516.224173:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06506a00. 00000100:00000001:3.0:1773941516.224182:0:14729:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941516.224183:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941516.224186:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1773941516.224188:0:14729:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941516.224189:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941516.224199:0:14729:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941516.224202:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941516.224203:0:14729:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941516.224212:0:14729:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941516.224212:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1773941516.224224:0:14729:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941516.224224:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941516.224228:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941516.224232:0:14729:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941516.224234:0:14729:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:3.0:1773941516.224237:0:14729:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:3.0:1773941516.224251:0:14729:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:3.0:1773941516.224253:0:14729:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941516.224255:0:14729:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:3.0:1773941516.224261:0:14729:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:3.0:1773941516.224266:0:14729:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941516.224268:0:14729:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941516.224269:0:14729:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941516.224271:0:14729:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941516.224272:0:14729:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:2.1:1773941516.227009:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d21ec1400. 00010000:00000010:2.1:1773941516.227020:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d00031800. 00000020:00000001:2.0:1773941516.283137:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941516.283143:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941516.283146:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1773941517.307196:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941517.307203:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941517.307208:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941517.499065:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294907817, at 4294907840 00000100:00000040:0.0:1773941517.499065:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294907817, at 4294907840 10000000:00000001:2.0:1773941517.499066:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 00000100:00000010:1.0:1773941517.499078:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d08bbbe40. 00000100:00000010:0.0:1773941517.499080:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d08bbbd80. 10000000:00000040:2.0:1773941517.499086:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941517.499089:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941517.499094:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941517.499096:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941517.499097:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:2.0:1773941517.499099:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:2.0:1773941517.499107:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:2.0:1773941517.499110:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:2.0:1773941517.499112:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:2.0:1773941517.499113:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:2.0:1773941517.499120:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06506a00. 00000020:00000040:2.0:1773941517.499124:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941517.499128:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941517.499130:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941517.499134:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941517.499140:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d08a4f400. 00000100:00000001:2.0:1773941517.499145:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941517.499150:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:2.0:1773941517.499152:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:2.0:1773941517.499157:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d071ee100. 00010000:00000001:2.0:1773941517.499161:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:2.0:1773941517.499164:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d089dfa00. 00000020:00000001:2.0:1773941517.499167:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:2.0:1773941517.499170:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d089dfa00 with handle 0x8e86767622e292f to hash 00000020:00000001:2.0:1773941517.499171:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:2.0:1773941517.499173:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941517.499175:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941517.499178:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941517.499180:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941517.499183:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e292f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941517.499193:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e292f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941517.499217:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e292f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941517.499223:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:2.0:1773941517.499225:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:2.0:1773941517.499232:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d09350540. 00000100:00000001:2.0:1773941517.499234:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330245121344 : -113743464430272 : ffff988d09350540) 00000100:00000001:2.0:1773941517.499237:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941517.499238:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584960 : -113743512966656 : ffff988d06506a00) 00000100:00000001:2.0:1773941517.499243:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941517.499248:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941517.499250:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:2.0:1773941517.499252:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941517.499263:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06506a00 x1860112291480832/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941517.499272:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941517.499275:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941517.499279:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941517.499281:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941517.499284:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941517.499288:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291480832:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941517.499293:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941517.499299:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941517.499301:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941517.499305:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941517.499310:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a5000. 02000000:00000001:2.0:1773941517.499312:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941517.499321:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941517.499326:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000400:00000010:2.0:1773941517.499332:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000200:2.0:1773941517.499339:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291480832, portal 25 00000100:00000001:2.0:1773941517.499341:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941517.499349:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584960 : -113743512966656 : ffff988d06506a00) 00000100:00000040:2.0:1773941517.499356:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06506a00 x1860112291480832/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941533 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941517.499366:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941517.499369:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:2.0:1773941517.499372:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045166e8. 00000100:00000200:2.0:1773941517.499376:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291480832, offset 0 00000400:00000200:2.0:1773941517.499382:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941517.499391:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941517.499398:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941517.499403:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941517.499408:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941517.499412:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941517.499415:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941517.499419:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[34, 33] 00000400:00000200:2.0:1773941517.499423:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:2.0:1773941517.499427:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [145:145:256:4294967295] 192.168.204.124@tcp LPNI seq info [35:35:8:4294967295] 00000400:00000200:2.0:1773941517.499433:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:2.0:1773941517.499442:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:2.0:1773941517.499448:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:2.0:1773941517.499453:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0dbb00. 00000800:00000200:2.0:1773941517.499460:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:2.0:1773941517.499467:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:2.0:1773941517.499472:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0dbb00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:2.0:1773941517.499496:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941517.499501:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941517.499503:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294909095 00000100:00000010:2.0:1773941517.499508:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d08bbbcc0. 00000100:00000001:2.0:1773941517.499513:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1773941517.499519:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294909095 00000100:00000010:2.0:1773941517.499528:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d08bbb480. 00000100:00000001:2.0:1773941517.499531:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1773941517.499535:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941517.499538:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941517.499540:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941517.499542:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:2.0:1773941517.499545:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d09350540 going to sleep for 11 seconds 00000100:00000001:2.0:1773941517.499548:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941517.499552:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941517.499617:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941517.499622:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0dbb00. 00000400:00000200:1.0:1773941517.499627:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941517.499633:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941517.499639:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045166e8 00000400:00000010:1.0:1773941517.499642:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045166e8. 00000100:00000001:1.0:1773941517.499646:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941517.499650:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06506a00 x1860112291480832/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941533 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941517.499658:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941517.499661:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d06506a00 x1860112291480832/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941533 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941517.499667:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941517.499668:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941517.500784:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941517.500797:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941517.500799:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941517.500801:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941517.500806:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941517.500810:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941517.500815:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941517.500818:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941517.500821:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941517.500825:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324604500 00000400:00000200:0.0:1773941517.500829:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941517.500834:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xc19 [1] + 224 00000800:00000001:0.0:1773941517.500839:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941517.500852:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941517.500855:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941517.500857:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941517.500875:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941517.500880:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06506a00 x1860112291480832/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941533 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941517.500887:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d06506a00 x1860112291480832/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941533 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941517.500901:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941517.500907:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941517.500910:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941517.500943:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941517.500950:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941517.500953:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941517.500955:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941517.500977:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941517.500980:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941517.500983:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941517.500989:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06506a00 x1860112291480832/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941533 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941517.501000:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06506a00 x1860112291480832/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941533 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941517.501008:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941517.501010:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941517.501012:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941517.501015:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941517.501018:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941517.501020:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941517.501023:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941517.501030:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=18 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941517.501034:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941517.501038:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:2.0:1773941517.501042:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:2.0:1773941517.501045:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941517.501049:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:2.0:1773941517.501052:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1773941517.501063:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1773941517.501068:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1773941517.501072:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941517.501075:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941517.501076:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941517.501078:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:2.0:1773941517.501083:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d06506a00 x1860112291480832/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941533 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941517.501092:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941517.501094:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941517.501096:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941517.501099:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941517.501113:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:2.0:1773941517.501115:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941517.501117:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941517.501119:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941517.501120:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941517.501122:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941517.501123:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941517.501125:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1773941517.501128:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1773941517.501132:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941517.501135:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 239 00000400:00000200:2.0:1773941517.501140:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941517.501144:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:2.0:1773941517.501149:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941517.501152:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941517.501155:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941517.501157:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:2.0:1773941517.501161:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941517.501163:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941517.501166:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941517.501174:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941517.501177:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941517.501180:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:2.0:1773941517.501184:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941517.501187:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941517.501189:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941517.501191:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:2.0:1773941517.501195:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:2.0:1773941517.501198:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:2.0:1773941517.501201:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941517.501204:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941517.501208:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941517.501210:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941517.501213:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941517.501215:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941517.501220:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06507b80. 00000020:00000040:2.0:1773941517.501223:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:2.0:1773941517.501226:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941517.501228:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941517.501230:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941517.501232:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941517.501234:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941517.501239:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d035a4c00. 00000100:00000001:2.0:1773941517.501243:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941517.501250:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d06507b80 x1860112291480960/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941517.501259:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941517.501262:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941517.501267:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000df552709] to pc [ptlrpcd_rcv+-1] req@ffff988d06507b80 x1860112291480960/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941517.501293:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941517.501297:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:2.0:1773941517.501307:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941517.501310:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941517.501312:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941517.501314:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941517.501317:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d09350540 going to sleep for 1 seconds 00000100:00000001:2.0:1773941517.501319:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941517.501322:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941517.501329:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06506a00 x1860112291480832/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941533 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941517.501337:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941517.501338:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941517.501342:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941517.501344:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941517.501345:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1773941517.501349:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:3.0:1773941517.501352:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06507b80 x1860112291480960/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941517.501353:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941517.501364:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:2.0:1773941517.501365:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06506a00 x1860112291480832/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941533 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941517.501366:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941517.501370:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941517.501373:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941517.501377:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941517.501385:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06507b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291480960:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941517.501389:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00100000:2.0:1773941517.501390:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291480832:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941517.501397:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1773941517.501400:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1773941517.501401:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:3.0:1773941517.501402:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941517.501407:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1773941517.501408:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 02000000:00000010:3.0:1773941517.501410:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03520c00. 00000100:00000001:2.0:1773941517.501412:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1773941517.501413:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941517.501419:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000040:2.0:1773941517.501420:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06506a00 x1860112291480832/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941533 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000010:3.0:1773941517.501422:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00000400:00000010:3.0:1773941517.501427:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00000100:00000200:3.0:1773941517.501432:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291480960, portal 25 00000100:00000001:2.0:1773941517.501434:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941517.501435:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941517.501438:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196589440 : -113743512962176 : ffff988d06507b80) 00000100:00000010:2.0:1773941517.501440:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d09350540. 00000100:00000040:3.0:1773941517.501444:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06507b80 x1860112291480960/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941527 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941517.501446:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:2.0:1773941517.501450:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941517.501452:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:2.0:1773941517.501457:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000040:3.0:1773941517.501458:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:3.0:1773941517.501460:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac908. 00000100:00000200:3.0:1773941517.501463:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291480960, offset 0 00010000:00000001:2.0:1773941517.501463:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:2.0:1773941517.501467:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:3.0:1773941517.501471:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000020:00000040:2.0:1773941517.501472:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089dfa00 refcount=4 00000400:00000200:3.0:1773941517.501476:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000020:00000001:2.0:1773941517.501477:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00000400:00000200:3.0:1773941517.501480:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941517.501484:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941517.501488:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00000001:2.0:1773941517.501489:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00000400:00000200:3.0:1773941517.501492:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941517.501495:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00010000:2.0:1773941517.501496:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e292f lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941517.501499:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[34, 35] 00000400:00000200:3.0:1773941517.501508:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941517.501513:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [146:146:256:4294967295] 192.168.204.114@tcp LPNI seq info [36:36:8:4294967295] 00010000:00000001:2.0:1773941517.501516:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000200:3.0:1773941517.501517:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:2.0:1773941517.501525:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e292f lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941517.501526:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:3.0:1773941517.501534:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:3.0:1773941517.501539:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74800. 00010000:00000001:2.0:1773941517.501544:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000800:00000200:3.0:1773941517.501548:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00010000:00010000:2.0:1773941517.501550:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e292f lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941517.501562:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00010000:00000001:2.0:1773941517.501564:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000200:3.0:1773941517.501566:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74800 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:2.0:1773941517.501568:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:2.0:1773941517.501572:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e292f lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941517.501579:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941517.501579:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941517.501582:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941517.501585:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:3.0:1773941517.501589:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294909097 00010000:00000001:2.0:1773941517.501589:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:2.0:1773941517.501595:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e292f lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941517.501596:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294909095 00000100:00000001:3.0:1773941517.501603:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294909097 00000100:00000001:3.0:1773941517.501609:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294909095 00010000:00010000:2.0:1773941517.501609:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d089dfa00 already unused, calling callback (ffffffffc12b4c20) 00000100:00000001:3.0:1773941517.501613:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 10000000:00000001:2.0:1773941517.501615:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000001:3.0:1773941517.501616:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941517.501620:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 10000000:00010000:2.0:1773941517.501621:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e292f lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941517.501623:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941517.501631:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941517.501634:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00010000:00000001:2.0:1773941517.501634:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000100:00000001:3.0:1773941517.501639:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:2.0:1773941517.501639:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000001:3.0:1773941517.501642:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1773941517.501643:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:3.0:1773941517.501644:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941517.501647:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089dfa00 refcount=5 00000100:00000001:3.0:1773941517.501648:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941517.501658:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941517.501661:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941517.501665:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000800:00000200:1.0:1773941517.501668:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00010000:2.0:1773941517.501669:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e292f lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000010:1.0:1773941517.501675:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74800. 00000400:00000200:1.0:1773941517.501682:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 10000000:00000001:2.0:1773941517.501683:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:2.0:1773941517.501688:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e292f lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:1.0:1773941517.501691:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941517.501698:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac908 10000000:01000000:2.0:1773941517.501703:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000010:1.0:1773941517.501703:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac908. 10000000:00000040:2.0:1773941517.501710:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000100:00000001:1.0:1773941517.501710:0:2393:0:(events.c:34:request_out_callback()) Process entered 10000000:00000001:2.0:1773941517.501713:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941517.501717:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000100:00000200:1.0:1773941517.501718:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06507b80 x1860112291480960/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941527 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1773941517.501722:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:2.0:1773941517.501727:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1773941517.501730:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000100:00000001:1.0:1773941517.501733:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00010000:00000001:2.0:1773941517.501735:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941517.501740:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941517.501740:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06507b80 x1860112291480960/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941527 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1773941517.501746:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d089dfa00 with handle 0x8e86767622e292f from hash 00010000:00000001:2.0:1773941517.501752:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1773941517.501756:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:1.0:1773941517.501756:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941517.501760:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:1.0:1773941517.501761:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:2.0:1773941517.501763:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1773941517.501767:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:2.0:1773941517.501769:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:2.0:1773941517.501772:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941517.501774:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941517.501776:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941517.501778:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941517.501783:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e292f lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941517.501790:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941517.501797:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941517.501799:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:2.0:1773941517.501801:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:2.0:1773941517.501802:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941517.501804:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941517.501805:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1773941517.501808:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e292f lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:2.0:1773941517.501816:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d071ee100 count: 0 00010000:00000001:2.0:1773941517.501825:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941517.501827:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941517.501831:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941517.501836:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06506a00 x1860112291480832/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941533 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941517.501844:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941517.501847:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941517.501850:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a5000. 02000000:00000001:2.0:1773941517.501853:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941517.501855:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941517.501857:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941517.501859:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941517.501875:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d08a4f400. 02000000:00000001:2.0:1773941517.501880:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941517.501882:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941517.501884:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06506a00. 00000100:00000001:2.0:1773941517.501887:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941517.501888:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:2.0:1773941517.501890:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:2.0:1773941517.501893:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:2.0:1773941517.501896:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:2.0:1773941517.501900:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d05437280. 00000020:00000010:2.0:1773941517.501904:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d08a4f400. 00000020:00000010:2.0:1773941517.501909:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d08a4fe00. 00000040:00000010:2.0:1773941517.501914:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d03356200. 00000020:00000010:2.0:1773941517.501919:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941517.501930:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d0381f200. 80000000:00000010:2.0:1773941517.501934:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d08bbb0c0. 00040000:00000010:2.0:1773941517.501938:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d0381e800. 20000000:00000010:2.0:1773941517.501942:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d0381e000. 10000000:00000040:2.0:1773941517.501947:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:2.0:1773941517.501950:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:2.0:1773941517.501954:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d08a4fe00. 00000040:00000010:2.0:1773941517.501959:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d03356200. 00000020:00000010:2.0:1773941517.501994:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941517.501999:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d0381f200. 80000000:00000010:2.0:1773941517.502003:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d08bbb0c0. 00040000:00000010:2.0:1773941517.502007:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d0381e800. 20000000:00000010:2.0:1773941517.502011:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d0381e000. 00000020:00000010:2.0:1773941517.502014:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d08a4f400. 10000000:00000010:2.0:1773941517.502018:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d05437280. 10000000:01000000:2.0:1773941517.502022:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:2.0:1773941517.502026:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:2.0:1773941517.502029:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:2.0:1773941517.502031:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:2.0:1773941517.502033:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:2.0:1773941517.502036:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941517.502038:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941517.503284:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941517.503293:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941517.503296:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941517.503300:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941517.503304:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941517.503310:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941517.503317:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941517.503322:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941517.503327:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941517.503331:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324604580 00000400:00000200:0.0:1773941517.503335:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941517.503340:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xc21 [1] + 0 00000800:00000001:0.0:1773941517.503346:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941517.503353:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941517.503356:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941517.503360:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941517.503364:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941517.503370:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06507b80 x1860112291480960/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941527 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941517.503381:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d06507b80 x1860112291480960/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941527 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941517.503414:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941517.503421:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941517.503425:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941517.503435:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941517.503439:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941517.503442:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941517.503444:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941517.503445:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941517.503450:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941517.503451:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941517.503453:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941517.503458:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06507b80 x1860112291480960/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941527 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941517.503466:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06507b80 x1860112291480960/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941527 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941517.503471:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941517.503472:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941517.503474:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941517.503491:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941517.503493:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941517.503494:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941517.503496:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941517.503499:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=18 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941517.503502:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941517.503503:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941517.503505:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941517.503507:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941517.503508:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941517.503510:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941517.503513:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06507b80 x1860112291480960/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941527 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941517.503518:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941517.503520:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941517.503521:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941517.503523:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941517.503527:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941517.503528:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941517.503529:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941517.503531:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941517.503532:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941517.503534:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941517.503536:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941517.503538:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941517 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941517.503540:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941517.503545:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941517.503546:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941517.503547:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941517.503549:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941517.503550:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941517.503552:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941517.503554:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941517.503556:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941517.503562:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941517.503565:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941517.503566:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941517.503777:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941517.503781:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941517.503787:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06507b80 x1860112291480960/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941527 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941517.503789:0:14730:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:2.0:1773941517.503797:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06507b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291480960:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00080000:3.0:1773941517.503799:0:14730:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:3.0:1773941517.503805:0:14730:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941517.503805:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941517.503809:0:14730:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:2.0:1773941517.503810:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 10000000:01000000:3.0:1773941517.503811:0:14730:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:3.0:1773941517.503815:0:14730:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000040:2.0:1773941517.503816:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06507b80 x1860112291480960/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941527 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000001:00000001:3.0:1773941517.503819:0:14730:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941517.503823:0:14730:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941517.503826:0:14730:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941517.503826:0:14730:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941517.503827:0:14730:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941517.503828:0:14730:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941517.503831:0:14730:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941517.503835:0:14730:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:3.0:1773941517.503840:0:14730:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:3.0:1773941517.503843:0:14730:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941517.503846:0:14730:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941517.503847:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00080000:3.0:1773941517.503848:0:14730:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:3.0:1773941517.503852:0:14730:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941517.503853:0:14730:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 02000000:00000001:2.0:1773941517.503853:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 10000000:00000040:3.0:1773941517.503858:0:14730:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 02000000:00000010:2.0:1773941517.503870:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03520c00. 02000000:00000001:2.0:1773941517.503876:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941517.503878:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941517.503880:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941517.503883:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941517.503885:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d035a4c00. 00000100:00080000:3.0:1773941517.503886:0:14730:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:3.0:1773941517.503889:0:14730:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941517.503889:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941517.503893:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1773941517.503894:0:14730:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000010:2.0:1773941517.503898:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06507b80. 00000100:02000000:3.0:1773941517.503903:0:14730:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:2.0:1773941517.503904:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941517.503906:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941517.503909:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941517.503910:0:14730:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941517.503913:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:3.0:1773941517.503916:0:14730:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:2.0:1773941517.503916:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1773941517.503921:0:14730:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941517.503921:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941517.503924:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941517.503928:0:14730:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941517.503931:0:14730:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:2.1:1773941517.507999:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d071ee100. 00010000:00000010:2.1:1773941517.508007:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d089dfa00. 00000020:00000001:2.0:1773941518.331135:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941518.331141:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941518.331145:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941518.779055:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294909097, at 4294909120 00000100:00000040:0.0:1773941518.779055:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294909097, at 4294909120 00000100:00000010:0.0:1773941518.779073:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d08bbbcc0. 00000100:00000010:1.0:1773941518.779076:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d08bbb480. 10000000:00000001:2.0:1773941519.355151:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:2.0:1773941519.355159:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941519.355164:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941519.355166:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941519.355167:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941519.355169:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:2.0:1773941519.355171:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:2.0:1773941519.355173:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:2.0:1773941519.355176:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:2.0:1773941519.355178:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:2.0:1773941519.355180:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000020:00000001:1.0:1773941519.355181:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1773941519.355184:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941519.355192:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06507b80. 00000020:00000040:2.0:1773941519.355198:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00010000:00010000:1.0:1773941519.355202:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1773941519.355205:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941519.355210:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941519.355223:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941519.355232:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d0381e000. 00000100:00000001:2.0:1773941519.355239:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941519.355245:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:2.0:1773941519.355248:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:2.0:1773941519.355254:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d071ee100. 00010000:00000001:2.0:1773941519.355259:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:2.0:1773941519.355262:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d089dfa00. 00000020:00000001:2.0:1773941519.355265:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:2.0:1773941519.355268:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d089dfa00 with handle 0x8e86767622e2936 to hash 00000020:00000001:2.0:1773941519.355270:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:2.0:1773941519.355271:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941519.355273:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941519.355276:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941519.355279:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941519.355282:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2936 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941519.355289:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2936 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941519.355298:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2936 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941519.355303:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:2.0:1773941519.355304:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:2.0:1773941519.355309:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d09350540. 00000100:00000001:2.0:1773941519.355310:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330245121344 : -113743464430272 : ffff988d09350540) 00000100:00000001:2.0:1773941519.355312:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941519.355313:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196589440 : -113743512962176 : ffff988d06507b80) 00000100:00000001:2.0:1773941519.355317:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941519.355321:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941519.355323:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:2.0:1773941519.355324:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941519.355339:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06507b80 x1860112291481088/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941519.355347:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941519.355348:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941519.355352:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941519.355354:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941519.355355:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941519.355358:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06507b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291481088:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941519.355361:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941519.355365:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941519.355367:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941519.355369:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941519.355372:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a4c00. 02000000:00000001:2.0:1773941519.355374:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941519.355380:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941519.355383:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000400:00000010:2.0:1773941519.355388:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000200:2.0:1773941519.355393:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291481088, portal 25 00000100:00000001:2.0:1773941519.355394:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941519.355395:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196589440 : -113743512962176 : ffff988d06507b80) 00000100:00000040:2.0:1773941519.355399:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06507b80 x1860112291481088/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941535 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941519.355405:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941519.355406:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:2.0:1773941519.355408:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516990. 00000100:00000200:2.0:1773941519.355410:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291481088, offset 0 00000400:00000200:2.0:1773941519.355415:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941519.355421:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941519.355427:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941519.355431:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941519.355434:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941519.355438:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941519.355444:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941519.355446:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[36, 35] 00000400:00000200:2.0:1773941519.355449:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:2.0:1773941519.355452:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [147:147:256:4294967295] 192.168.204.124@tcp LPNI seq info [37:37:8:4294967295] 00000400:00000200:2.0:1773941519.355455:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:2.0:1773941519.355472:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:2.0:1773941519.355477:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:2.0:1773941519.355481:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0dbb00. 00000800:00000200:2.0:1773941519.355487:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:2.0:1773941519.355493:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:2.0:1773941519.355497:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0dbb00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:2.0:1773941519.355509:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941519.355512:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941519.355515:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294910951 00000100:00000010:2.0:1773941519.355518:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d08bbb0c0. 00000100:00000001:2.0:1773941519.355525:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1773941519.355531:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294910951 00000100:00000010:2.0:1773941519.355533:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d08bbb780. 00000100:00000001:2.0:1773941519.355535:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1773941519.355538:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941519.355540:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941519.355541:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941519.355543:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:2.0:1773941519.355544:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d09350540 going to sleep for 11 seconds 00000100:00000001:2.0:1773941519.355546:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941519.355549:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941519.355626:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941519.355630:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0dbb00. 00000400:00000200:1.0:1773941519.355635:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941519.355640:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941519.355644:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516990 00000400:00000010:1.0:1773941519.355646:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516990. 00000100:00000001:1.0:1773941519.355649:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941519.355653:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06507b80 x1860112291481088/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941535 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941519.355661:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941519.355664:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d06507b80 x1860112291481088/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941535 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941519.355670:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941519.355671:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941519.356707:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941519.356728:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941519.356732:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941519.356734:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941519.356744:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941519.356753:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941519.356765:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941519.356771:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941519.356777:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941519.356782:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324604600 00000400:00000200:0.0:1773941519.356787:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941519.356794:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xc29 [1] + 224 00000800:00000001:0.0:1773941519.356803:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941519.356816:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941519.356819:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941519.356853:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941519.356862:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941519.356882:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06507b80 x1860112291481088/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941535 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941519.356896:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d06507b80 x1860112291481088/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941535 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941519.356916:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941519.356925:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941519.356957:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941519.357031:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941519.357037:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941519.357038:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941519.357041:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941519.357044:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941519.357046:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941519.357054:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941519.357060:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06507b80 x1860112291481088/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941535 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941519.357068:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06507b80 x1860112291481088/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941535 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941519.357074:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941519.357075:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941519.357077:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941519.357079:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941519.357081:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941519.357082:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941519.357084:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941519.357089:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=20 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941519.357092:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941519.357095:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:2.0:1773941519.357098:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:2.0:1773941519.361572:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941519.361578:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:2.0:1773941519.361581:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1773941519.361585:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1773941519.361589:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1773941519.366237:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941519.366240:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941519.366242:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941519.366244:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:2.0:1773941519.366250:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d06507b80 x1860112291481088/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941535 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941519.366259:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941519.366260:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941519.366262:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941519.366266:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941519.366268:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:2.0:1773941519.366279:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941519.366281:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941519.366282:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941519.366282:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941519.366284:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941519.366285:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941519.366287:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1773941519.366288:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1773941519.366291:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941519.366293:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 240 00000400:00000200:2.0:1773941519.366299:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941519.366302:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:2.0:1773941519.366306:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941519.366308:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941519.366311:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941519.366312:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:2.0:1773941519.366315:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941519.366317:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941519.366319:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941519.366321:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941519.366338:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941519.366339:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:2.0:1773941519.366341:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941519.366343:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941519.366344:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941519.366345:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:2.0:1773941519.366348:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:2.0:1773941519.366350:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:2.0:1773941519.366352:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941519.366354:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941519.366356:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941519.366358:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941519.366363:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941519.366365:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941519.366368:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06506a00. 00000020:00000040:2.0:1773941519.366370:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:2.0:1773941519.366373:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941519.366374:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941519.366388:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941519.366390:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941519.366393:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941519.366398:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d035a5000. 00000100:00000001:2.0:1773941519.366402:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941519.366406:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d06506a00 x1860112291481216/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941519.366413:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941519.366416:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941519.366419:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6b603d8] to pc [ptlrpcd_rcv+-1] req@ffff988d06506a00 x1860112291481216/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941519.366437:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941519.366440:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:2.0:1773941519.366441:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941519.366444:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941519.366458:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941519.366460:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941519.366462:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d09350540 going to sleep for 1 seconds 00000100:00000001:2.0:1773941519.366463:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941519.366465:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941519.366468:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06507b80 x1860112291481088/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941535 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941519.366475:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941519.366478:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941519.366481:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941519.366483:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941519.366489:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1773941519.366490:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06507b80 x1860112291481088/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941535 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941519.366494:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941519.366504:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941519.366512:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06506a00 x1860112291481216/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941519.366524:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06507b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291481088:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941519.366537:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941519.366538:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941519.366540:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941519.366545:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1773941519.366545:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:3.0:1773941519.366548:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941519.366551:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 02000000:00000001:3.0:1773941519.366552:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941519.366556:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00100000:3.0:1773941519.366557:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291481216:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941519.366562:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000040:2.0:1773941519.366566:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06507b80 x1860112291481088/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941535 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1773941519.366567:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941519.366569:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941519.366574:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941519.366589:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03523000. 02000000:00000001:3.0:1773941519.366594:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941519.366595:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941519.366600:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000010:2.0:1773941519.366600:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d09350540. 00000400:00000010:3.0:1773941519.366607:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00000100:00000001:2.0:1773941519.366607:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:2.0:1773941519.366611:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000010:3.0:1773941519.366614:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00010000:00000001:2.0:1773941519.366615:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:2.0:1773941519.366620:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000200:3.0:1773941519.366622:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291481216, portal 25 00000020:00000001:2.0:1773941519.366622:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:3.0:1773941519.366627:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000020:00000040:2.0:1773941519.366627:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089dfa00 refcount=4 00000100:00000001:3.0:1773941519.366631:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584960 : -113743512966656 : ffff988d06506a00) 00000020:00000001:2.0:1773941519.366631:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941519.366636:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00000100:00000040:3.0:1773941519.366640:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06506a00 x1860112291481216/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941529 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:2.0:1773941519.366642:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2936 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941519.366670:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941519.366671:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941519.366676:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00010000:2.0:1773941519.366677:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2936 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:3.0:1773941519.366681:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac908. 00010000:00000001:2.0:1773941519.366683:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000200:3.0:1773941519.366687:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291481216, offset 0 00010000:00010000:2.0:1773941519.366688:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2936 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941519.366697:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:3.0:1773941519.366700:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00000001:2.0:1773941519.366705:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:2.0:1773941519.366708:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2936 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941519.366713:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:2.0:1773941519.366718:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941519.366722:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941519.366724:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:2.0:1773941519.366726:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:2.0:1773941519.366730:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2936 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941519.366733:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00010000:2.0:1773941519.366740:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d089dfa00 already unused, calling callback (ffffffffc12b4c20) 00000400:00000200:3.0:1773941519.366741:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 10000000:00000001:2.0:1773941519.366743:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:2.0:1773941519.366747:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2936 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941519.366748:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:2.0:1773941519.366752:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:2.0:1773941519.366754:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:2.0:1773941519.366755:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941519.366758:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089dfa00 refcount=5 00000400:00000200:3.0:1773941519.366760:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000020:00000001:2.0:1773941519.366761:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941519.366766:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00000400:00000200:3.0:1773941519.366767:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[36, 37] 00010000:00000001:2.0:1773941519.366771:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:2.0:1773941519.366775:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2936 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941519.366788:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 10000000:00000001:2.0:1773941519.366789:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:3.0:1773941519.366794:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [148:148:256:4294967295] 192.168.204.114@tcp LPNI seq info [38:38:8:4294967295] 10000000:00010000:2.0:1773941519.366795:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2936 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941519.366803:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 10000000:01000000:2.0:1773941519.366810:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:2.0:1773941519.366817:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000400:00000200:3.0:1773941519.366818:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 10000000:00000001:2.0:1773941519.366821:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941519.366839:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00010000:00010000:2.0:1773941519.366840:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000800:00000010:3.0:1773941519.366844:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74c00. 00010000:00000001:2.0:1773941519.366845:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000800:00000200:3.0:1773941519.366852:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:3.0:1773941519.366859:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00010000:00000001:2.0:1773941519.366859:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1773941519.366863:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000800:00000200:3.0:1773941519.366865:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74c00 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:2.0:1773941519.366868:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941519.366872:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941519.366883:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d089dfa00 with handle 0x8e86767622e2936 from hash 00010000:00000001:2.0:1773941519.366887:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000100:00000001:3.0:1773941519.366888:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941519.366889:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941519.366893:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:3.0:1773941519.366895:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:2.0:1773941519.366896:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000100:00000001:3.0:1773941519.366899:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294910962 00010000:00000001:2.0:1773941519.366900:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:2.0:1773941519.366904:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:2.0:1773941519.366907:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941519.366908:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:3.0:1773941519.366909:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294910951 00010000:00000001:2.0:1773941519.366911:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941519.366915:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941519.366917:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294910962 00010000:00010000:2.0:1773941519.366918:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2936 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941519.366924:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294910951 00010000:00000001:2.0:1773941519.366926:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941519.366929:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:2.0:1773941519.366929:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941519.366932:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000100:00000001:3.0:1773941519.366934:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941519.366935:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:2.0:1773941519.366938:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941519.366939:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941519.366940:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:3.0:1773941519.366943:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1773941519.366943:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1773941519.366945:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2936 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941519.366950:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941519.366953:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1773941519.366955:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000040:2.0:1773941519.366955:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d071ee100 count: 0 00000100:00000001:3.0:1773941519.366957:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941519.366980:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941519.366982:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941519.367010:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941519.367013:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941519.367017:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000800:00000200:1.0:1773941519.367023:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000040:2.0:1773941519.367025:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06507b80 x1860112291481088/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941535 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000800:00000010:1.0:1773941519.367030:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74c00. 00000100:00000001:2.0:1773941519.367036:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000400:00000200:1.0:1773941519.367039:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:2.0:1773941519.367040:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941519.367049:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a4c00. 00000400:00000200:1.0:1773941519.367050:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 02000000:00000001:2.0:1773941519.367052:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941519.367056:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941519.367058:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000400:00000200:1.0:1773941519.367058:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac908 00000020:00000001:2.0:1773941519.367079:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 00000400:00000010:1.0:1773941519.367080:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac908. 02000000:00000010:2.0:1773941519.367082:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d0381e000. 02000000:00000001:2.0:1773941519.367088:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1773941519.367088:0:2393:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:2.0:1773941519.367092:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941519.367094:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06507b80. 00000100:00000001:2.0:1773941519.367097:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941519.367099:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:1.0:1773941519.367099:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06506a00 x1860112291481216/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941529 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000001:2.0:1773941519.367102:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:2.0:1773941519.367104:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:2.0:1773941519.367106:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:2.0:1773941519.367108:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d05437280. 00000020:00000010:2.0:1773941519.367111:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d0381e000. 00000020:00000010:2.0:1773941519.367117:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d0381e800. 00000040:00000010:2.0:1773941519.367122:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d0381f200. 00000020:00000010:2.0:1773941519.367128:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d09305000. 00000100:00000001:1.0:1773941519.367128:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000020:00000010:2.0:1773941519.367135:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d03356200. 00000100:00000040:1.0:1773941519.367138:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06506a00 x1860112291481216/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941529 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 80000000:00000010:2.0:1773941519.367139:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d08bbb540. 00040000:00000010:2.0:1773941519.367146:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d08a4f400. 20000000:00000010:2.0:1773941519.367152:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d08a4fe00. 00000100:00000001:1.0:1773941519.367154:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 10000000:00000040:2.0:1773941519.367157:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 00000100:00000001:1.0:1773941519.367158:0:2393:0:(events.c:68:request_out_callback()) Process leaving 10000000:00000001:2.0:1773941519.367160:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:2.0:1773941519.367165:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d0381e800. 00000040:00000010:2.0:1773941519.367168:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d0381f200. 00000020:00000010:2.0:1773941519.367171:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941519.367185:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d03356200. 80000000:00000010:2.0:1773941519.367188:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d08bbb540. 00040000:00000010:2.0:1773941519.367192:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d08a4f400. 20000000:00000010:2.0:1773941519.367211:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d08a4fe00. 00000020:00000010:2.0:1773941519.367214:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0381e000. 10000000:00000010:2.0:1773941519.367215:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d05437280. 10000000:01000000:2.0:1773941519.367217:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:2.0:1773941519.367223:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:2.0:1773941519.367224:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:2.0:1773941519.367226:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:2.0:1773941519.367227:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:2.0:1773941519.367229:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941519.367230:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941519.372445:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941519.372468:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941519.372472:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941519.372475:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941519.372481:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941519.372487:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941519.372494:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941519.372498:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941519.372503:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941519.372507:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324604680 00000400:00000200:0.0:1773941519.372512:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941519.372517:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xc31 [1] + 0 00000800:00000001:0.0:1773941519.372524:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941519.372533:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941519.372536:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941519.372540:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941519.372546:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941519.372552:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06506a00 x1860112291481216/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941529 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941519.372563:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d06506a00 x1860112291481216/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941529 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941519.372578:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941519.372585:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941519.372588:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941519.372616:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941519.372621:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941519.372626:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941519.372627:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941519.372630:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941519.372639:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941519.372641:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941519.372643:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941519.372648:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06506a00 x1860112291481216/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941529 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941519.372661:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06506a00 x1860112291481216/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941529 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941519.372669:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941519.372671:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941519.372674:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941519.372677:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941519.372679:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941519.372682:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941519.372685:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941519.372691:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=20 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941519.372695:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941519.372698:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941519.372699:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941519.372702:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941519.372704:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941519.372706:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941519.372712:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06506a00 x1860112291481216/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941529 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941519.372720:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941519.372723:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941519.372725:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941519.372727:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941519.372733:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941519.372736:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941519.372738:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941519.372740:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941519.372743:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941519.372746:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941519.372749:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941519.372760:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941519 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941519.372763:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941519.372770:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941519.372772:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941519.372774:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941519.372777:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941519.372779:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941519.372783:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941519.372785:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941519.372788:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941519.372792:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941519.372796:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941519.372798:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941519.373029:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941519.373033:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941519.373039:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06506a00 x1860112291481216/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941529 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941519.373047:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291481216:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941519.373049:0:14731:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941519.373052:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941519.373055:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941519.373058:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06506a00 x1860112291481216/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941529 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1773941519.373062:0:14731:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:3.0:1773941519.373070:0:14731:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941519.373070:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941519.373073:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1773941519.373075:0:14731:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000010:2.0:1773941519.373077:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03523000. 10000000:01000000:3.0:1773941519.373079:0:14731:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000001:2.0:1773941519.373082:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:3.0:1773941519.373084:0:14731:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:2.0:1773941519.373085:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941519.373088:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000001:00000001:3.0:1773941519.373090:0:14731:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000001:2.0:1773941519.373091:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941519.373095:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d035a5000. 00000001:00000001:3.0:1773941519.373097:0:14731:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941519.373098:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941519.373100:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941519.373103:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06506a00. 00000001:00000001:3.0:1773941519.373104:0:14731:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:2.0:1773941519.373106:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000001:00000001:3.0:1773941519.373107:0:14731:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941519.373109:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:3.0:1773941519.373111:0:14731:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941519.373112:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:3.0:1773941519.373118:0:14731:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941519.373119:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941519.373121:0:14731:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941519.373122:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941519.373126:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941519.373127:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1773941519.373128:0:14731:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:3.0:1773941519.373133:0:14731:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:3.0:1773941519.373136:0:14731:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941519.373138:0:14731:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1773941519.373141:0:14731:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:3.0:1773941519.373143:0:14731:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941519.373145:0:14731:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:3.0:1773941519.373146:0:14731:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:3.0:1773941519.373170:0:14731:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:3.0:1773941519.373173:0:14731:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941519.373175:0:14731:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:3.0:1773941519.373181:0:14731:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00010000:00000010:2.1:1773941519.373993:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d071ee100. 00010000:00000010:2.1:1773941519.373999:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d089dfa00. 00000100:00000001:3.0:1773941519.377480:0:14731:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941519.377484:0:14731:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941519.377485:0:14731:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941519.377487:0:14731:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941519.377488:0:14731:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941520.379077:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941520.379093:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941520.379096:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941520.635029:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294910962, at 4294910976 00000100:00000010:1.0:1773941520.635042:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d08bbb780. 00000100:00000040:0.0:1773941520.635046:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294910962, at 4294910976 00000100:00000010:0.0:1773941520.635056:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d08bbb0c0. 00000100:00000040:1.0:1773941520.635075:0:461:0:(pinger.c:229:ptlrpc_pinger_process_import()) c1c1ff21-5f68-4bf6-bdc4-258644b24465->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1773941520.635099:0:461:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1773941520.635106:0:461:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c2680. 00000020:00000040:1.0:1773941520.635111:0:461:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941520.635116:0:461:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941520.635119:0:461:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941520.635125:0:461:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941520.635138:0:461:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff988d02cfdd00. 00000100:00000001:1.0:1773941520.635144:0:461:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941520.635154:0:461:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging c1c1ff21-5f68-4bf6-bdc4-258644b24465->MGS req@ffff988d203c2680 x1860112291481344/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941520.635164:0:461:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941520.635167:0:461:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941520.635173:0:461:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000097d1012f] to pc [ptlrpcd_00_00+0] req@ffff988d203c2680 x1860112291481344/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941520.635195:0:461:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941520.635199:0:461:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (248) 00000100:00000001:1.0:1773941520.635210:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941520.635213:0:2401:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1773941520.635215:0:2402:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:0.0:1773941520.635218:0:2401:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941520.635219:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941520.635229:0:2402:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941520.635232:0:2402:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941520.635235:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941520.635237:0:2402:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941520.635239:0:2402:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941520.635245:0:2402:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c2680 x1860112291481344/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941520.635254:0:2402:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941520.635257:0:2402:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941520.635261:0:2402:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941520.635264:0:2402:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941520.635267:0:2402:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941520.635271:0:2402:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2402:1860112291481344:192.168.204.114@tcp:400:kworker.0 00000100:00000001:1.0:1773941520.635276:0:2402:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941520.635289:0:2402:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941520.635292:0:2402:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941520.635301:0:2402:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941520.635307:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff988d181fc200. 02000000:00000001:1.0:1773941520.635309:0:2402:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941520.635316:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941520.635320:0:2402:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c5d8. 00000400:00000010:1.0:1773941520.635325:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000100:00000200:1.0:1773941520.635331:0:2402:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112291481344, portal 25 00000100:00000001:1.0:1773941520.635332:0:2402:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941520.635334:0:2402:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631464576 : -113743078087040 : ffff988d203c2680) 00000100:00000040:1.0:1773941520.635338:0:2402:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c2680 x1860112291481344/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941536 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941520.635344:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941520.635346:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941520.635348:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de5d8. 00000100:00000200:1.0:1773941520.635350:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1860112291481344, offset 0 00000400:00000200:1.0:1773941520.635354:0:2402:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941520.635361:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941520.635367:0:2402:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941520.635371:0:2402:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941520.635374:0:2402:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941520.635378:0:2402:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941520.635380:0:2402:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941520.635382:0:2402:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[38, 37] 00000400:00000200:1.0:1773941520.635385:0:2402:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:1.0:1773941520.635388:0:2402:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [149:149:256:4294967295] 192.168.204.124@tcp LPNI seq info [39:39:8:4294967295] 00000400:00000200:1.0:1773941520.635391:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941520.635398:0:2402:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:1.0:1773941520.635402:0:2402:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:1.0:1773941520.635406:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfdf00. 00000800:00000200:1.0:1773941520.635411:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:1.0:1773941520.635421:0:2402:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:1.0:1773941520.635425:0:2402:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfdf00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1773941520.635437:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941520.635440:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941520.635443:0:2402:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294912231 00000100:00000010:1.0:1773941520.635446:0:2402:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f34e0. 00000100:00000001:1.0:1773941520.635450:0:2402:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1773941520.635453:0:2402:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294912231 00000100:00000010:1.0:1773941520.635456:0:2402:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3300. 00000100:00000001:1.0:1773941520.635458:0:2402:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1773941520.635461:0:2402:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941520.635463:0:2402:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941520.635466:0:2402:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941520.635468:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941520.635544:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941520.635548:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfdf00. 00000400:00000200:0.0:1773941520.635551:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941520.635556:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941520.635560:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de5d8 00000400:00000010:0.0:1773941520.635567:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de5d8. 00000100:00000001:0.0:1773941520.635570:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941520.635575:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c2680 x1860112291481344/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941536 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941520.635583:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941520.635586:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c2680 x1860112291481344/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941536 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941520.635591:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941520.635592:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941520.636719:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941520.636730:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941520.636732:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941520.636734:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941520.636739:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941520.636743:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941520.636747:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941520.636750:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941520.636753:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941520.636756:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324604700 00000400:00000200:0.0:1773941520.636759:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941520.636762:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xc39 [1] + 224 00000800:00000001:0.0:1773941520.636767:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941520.636773:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941520.636774:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941520.636778:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941520.636781:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941520.636784:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c2680 x1860112291481344/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941536 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941520.636791:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff988d203c2680 x1860112291481344/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941536 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941520.636817:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941520.636824:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941520.636832:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941520.636840:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941520.636842:0:2402:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941520.636847:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941520.636848:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941520.636850:0:2402:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:1.0:1773941520.636853:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:1.0:1773941520.636855:0:2402:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:1.0:1773941520.636856:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941520.636860:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c2680 x1860112291481344/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941536 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941520.636867:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c2680 x1860112291481344/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941536 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941520.636872:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941520.636873:0:2402:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941520.636875:0:2402:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941520.636877:0:2402:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941520.636895:0:2402:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941520.636897:0:2402:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941520.636899:0:2402:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941520.636905:0:2402:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=21 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941520.636915:0:2402:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941520.636918:0:2402:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1773941520.636920:0:2402:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941520.636923:0:2402:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941520.636925:0:2402:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941520.636929:0:2402:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941520.636932:0:2402:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941520.636935:0:2402:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941520.636937:0:2402:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941520.636939:0:2402:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941520.636940:0:2402:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941520.636943:0:2402:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d203c2680 x1860112291481344/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941536 ref 1 fl Rpc:RNQU/200/0 rc 0/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941520.636965:0:2402:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941520.636966:0:2402:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941520.636967:0:2402:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941520.636970:0:2402:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941520.636971:0:2402:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941520.636973:0:2402:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941520.636974:0:2402:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941520.636975:0:2402:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941520.636976:0:2402:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941520.636977:0:2402:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941520.636978:0:2402:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941520.636980:0:2402:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941520.636981:0:2402:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941520.636983:0:2402:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941520.636985:0:2402:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 242 00000400:00000200:1.0:1773941520.636988:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941520.637009:0:2402:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941520.637012:0:2402:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941520.637014:0:2402:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941520.637016:0:2402:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941520.637021:0:2402:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941520.637023:0:2402:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941520.637025:0:2402:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941520.637027:0:2402:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941520.637029:0:2402:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941520.637030:0:2402:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941520.637032:0:2402:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941520.637034:0:2402:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941520.637036:0:2402:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941520.637037:0:2402:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941520.637038:0:2402:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941520.637040:0:2402:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941520.637043:0:2402:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941520.637046:0:2402:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941520.637048:0:2402:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941520.637051:0:2402:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941520.637054:0:2402:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941520.637056:0:2402:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941520.637058:0:2402:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941520.637061:0:2402:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c1f80. 00000020:00000040:1.0:1773941520.637064:0:2402:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941520.637068:0:2402:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941520.637069:0:2402:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941520.637072:0:2402:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941520.637074:0:2402:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941520.637076:0:2402:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941520.637081:0:2402:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d1118b800. 00000100:00000001:1.0:1773941520.637085:0:2402:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941520.637091:0:2402:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c1f80 x1860112291481472/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941520.637100:0:2402:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941520.637103:0:2402:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941520.637113:0:2402:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000708a2e6d] to pc [ptlrpcd_rcv+-1] req@ffff988d203c1f80 x1860112291481472/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941520.637133:0:2402:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941520.637135:0:2402:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941520.637136:0:2402:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:1.0:1773941520.637140:0:2402:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c2680 x1860112291481344/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941536 ref 1 fl Rpc:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941520.637146:0:2402:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941520.637148:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941520.637149:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941520.637152:0:2402:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c2680 x1860112291481344/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941536 ref 1 fl Interpret:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941520.637157:0:2402:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2402:1860112291481344:192.168.204.114@tcp:400:kworker.0 00000100:00000001:1.0:1773941520.637161:0:2402:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941520.637163:0:2402:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941520.637165:0:2402:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c2680 x1860112291481344/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941536 ref 1 fl Complete:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941520.637170:0:2402:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1773941520.637174:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1773941520.637174:0:2402:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1773941520.637178:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 02000000:00000010:1.0:1773941520.637179:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff988d181fc200. 00000100:00000001:2.0:1773941520.637181:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1773941520.637183:0:2402:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941520.637186:0:2402:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941520.637189:0:2402:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000040:2.0:1773941520.637190:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c1f80 x1860112291481472/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1773941520.637191:0:2402:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941520.637195:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff988d02cfdd00. 02000000:00000001:1.0:1773941520.637201:0:2402:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941520.637203:0:2402:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1773941520.637205:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000010:1.0:1773941520.637206:0:2402:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c2680. 00000100:00000001:2.0:1773941520.637208:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941520.637210:0:2402:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 02000000:00000001:2.0:1773941520.637212:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1773941520.637213:0:2402:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1773941520.637216:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941520.637216:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1773941520.637219:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941520.637223:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291481472:192.168.204.114@tcp:250:ptlrpcd_00_01.0 00000100:00000001:2.0:1773941520.637227:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1773941520.637227:0:2402:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941520.637231:0:2402:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941520.637232:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1773941520.637235:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1773941520.637236:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941520.637239:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941520.637240:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941520.637245:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a4800. 02000000:00000001:2.0:1773941520.637249:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941520.637254:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941520.637258:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000400:00000010:2.0:1773941520.637263:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000200:2.0:1773941520.637267:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291481472, portal 25 00000100:00000001:2.0:1773941520.637269:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941520.637271:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631462784 : -113743078088832 : ffff988d203c1f80) 00000100:00000040:2.0:1773941520.637277:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c1f80 x1860112291481472/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941530 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941520.637284:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941520.637287:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:2.0:1773941520.637290:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045165d8. 00000100:00000200:2.0:1773941520.637293:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291481472, offset 0 00000400:00000200:2.0:1773941520.637299:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941520.637304:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941520.637315:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941520.637319:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941520.637322:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941520.637325:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941520.637328:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941520.637331:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[38, 39] 00000400:00000200:2.0:1773941520.637334:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:2.0:1773941520.637338:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [150:150:256:4294967295] 192.168.204.114@tcp LPNI seq info [40:40:8:4294967295] 00000400:00000200:2.0:1773941520.637342:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941520.637348:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:2.0:1773941520.637351:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:2.0:1773941520.637355:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0dbd00. 00000800:00000200:2.0:1773941520.637358:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:2.0:1773941520.637373:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:2.0:1773941520.637378:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0dbd00 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:2.0:1773941520.637388:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941520.637390:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941520.637392:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294912233 00000100:00000001:2.0:1773941520.637397:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294912231 00000100:00000001:2.0:1773941520.637399:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294912233 00000100:00000001:2.0:1773941520.637401:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294912231 00000100:00000001:2.0:1773941520.637403:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941520.637405:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941520.637407:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941520.637408:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941520.637412:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941520.637413:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941520.637415:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941520.637416:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941520.637418:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941520.637420:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941520.637478:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941520.637483:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0dbd00. 00000400:00000200:0.0:1773941520.637488:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941520.637493:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941520.637497:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045165d8 00000400:00000010:0.0:1773941520.637500:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045165d8. 00000100:00000001:0.0:1773941520.637503:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941520.637509:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c1f80 x1860112291481472/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941530 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941520.637518:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941520.637521:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c1f80 x1860112291481472/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941530 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941520.637529:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941520.637531:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941520.638842:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941520.638848:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941520.638849:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941520.638851:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941520.638854:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941520.638857:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941520.638866:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941520.638868:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941520.638871:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941520.638874:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324604780 00000400:00000200:0.0:1773941520.638876:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941520.638879:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xc49 [1] + 0 00000800:00000001:0.0:1773941520.638882:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941520.638887:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941520.638888:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941520.638891:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941520.638894:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941520.638897:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c1f80 x1860112291481472/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941530 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941520.638912:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c1f80 x1860112291481472/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941530 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941520.638926:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941520.638931:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941520.638933:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941520.638978:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941520.638981:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941520.638984:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941520.638985:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941520.638987:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941520.638990:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941520.638992:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941520.638993:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941520.638998:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c1f80 x1860112291481472/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941530 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941520.639006:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c1f80 x1860112291481472/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941530 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941520.639011:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941520.639012:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941520.639014:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941520.639015:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941520.639016:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941520.639017:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941520.639019:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941520.639021:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=21 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941520.639024:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941520.639026:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941520.639028:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941520.639031:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941520.639034:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941520.639037:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941520.639041:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c1f80 x1860112291481472/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941530 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941520.639049:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941520.639051:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941520.639053:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941520.639055:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941520.639066:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941520.639069:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941520.639071:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941520.639081:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941520.639083:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941520.639086:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941520.639088:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:2.0:1773941520.639093:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941520 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941520.639096:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941520.639101:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:2.0:1773941520.639104:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941520.639105:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941520.639107:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941520.639109:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941520.639113:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:2.0:1773941520.639115:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941520.639118:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941520.639121:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941520.639125:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:2.0:1773941520.639127:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941520.639335:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941520.639338:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941520.639343:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c1f80 x1860112291481472/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941530 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941520.639350:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291481472:192.168.204.114@tcp:250:ptlrpcd_00_01.0 00000100:00000001:2.0:1773941520.639353:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941520.639355:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941520.639358:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c1f80 x1860112291481472/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941530 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941520.639359:0:14733:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941520.639369:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00080000:0.0:1773941520.639370:0:14733:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 02000000:00000001:2.0:1773941520.639372:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941520.639375:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a4800. 00000100:00100000:0.0:1773941520.639375:0:14733:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:0.0:1773941520.639379:0:14733:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:2.0:1773941520.639380:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 10000000:01000000:0.0:1773941520.639382:0:14733:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:0.0:1773941520.639385:0:14733:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:2.0:1773941520.639389:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:0.0:1773941520.639390:0:14733:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000040:2.0:1773941520.639391:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000001:00000001:0.0:1773941520.639394:0:14733:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941520.639395:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941520.639399:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d1118b800. 00000001:00000001:0.0:1773941520.639399:0:14733:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941520.639401:0:14733:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941520.639403:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000001:00000001:0.0:1773941520.639405:0:14733:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941520.639406:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 10000000:00000001:0.0:1773941520.639408:0:14733:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941520.639409:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c1f80. 00000100:00000001:0.0:1773941520.639411:0:14733:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941520.639415:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00080000:0.0:1773941520.639416:0:14733:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941520.639417:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941520.639421:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941520.639421:0:14733:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:0.0:1773941520.639425:0:14733:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:2.0:1773941520.639427:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941520.639428:0:14733:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941520.639430:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941520.639431:0:14733:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941520.639433:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941520.639434:0:14733:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:2.0:1773941520.639435:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:01000000:0.0:1773941520.639437:0:14733:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941520.639440:0:14733:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:0.0:1773941520.639455:0:14733:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941520.639457:0:14733:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941520.639459:0:14733:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941520.639461:0:14733:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:0.0:1773941520.639465:0:14733:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941520.639467:0:14733:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941520.639469:0:14733:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:0.0:1773941520.639477:0:14733:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941520.639479:0:14733:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941521.091071:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:2.0:1773941521.091078:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941521.091081:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941521.091083:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941521.091084:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941521.091085:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:2.0:1773941521.091087:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:2.0:1773941521.091089:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:2.0:1773941521.091091:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:2.0:1773941521.091093:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:2.0:1773941521.091094:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:2.0:1773941521.091103:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06506a00. 00000020:00000040:2.0:1773941521.091107:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941521.091111:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941521.091113:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941521.091118:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941521.091125:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d08a4f400. 00000100:00000001:2.0:1773941521.091131:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941521.091135:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:2.0:1773941521.091137:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:2.0:1773941521.091143:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d071ee100. 00010000:00000001:2.0:1773941521.091147:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:2.0:1773941521.091149:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d089dfa00. 00000020:00000001:2.0:1773941521.091153:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:2.0:1773941521.091167:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d089dfa00 with handle 0x8e86767622e293d to hash 00000020:00000001:2.0:1773941521.091168:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:2.0:1773941521.091170:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941521.091172:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941521.091175:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941521.091178:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941521.091181:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e293d lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941521.091188:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e293d lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941521.091197:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e293d lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941521.091201:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:2.0:1773941521.091203:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:2.0:1773941521.091208:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d09350540. 00000100:00000001:2.0:1773941521.091210:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330245121344 : -113743464430272 : ffff988d09350540) 00000100:00000001:2.0:1773941521.091212:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941521.091214:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584960 : -113743512966656 : ffff988d06506a00) 00000100:00000001:2.0:1773941521.091217:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941521.091221:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941521.091222:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:2.0:1773941521.091224:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941521.091233:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06506a00 x1860112291481600/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941521.091240:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941521.091241:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941521.091244:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941521.091246:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941521.091248:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941521.091250:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291481600:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941521.091256:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941521.091261:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941521.091262:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941521.091265:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941521.091268:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a4800. 02000000:00000001:2.0:1773941521.091269:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941521.091275:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941521.091278:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000400:00000010:2.0:1773941521.091283:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000200:2.0:1773941521.091288:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291481600, portal 25 00000100:00000001:2.0:1773941521.091290:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941521.091291:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584960 : -113743512966656 : ffff988d06506a00) 00000100:00000040:2.0:1773941521.091294:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06506a00 x1860112291481600/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941537 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941521.091308:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941521.091310:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:2.0:1773941521.091315:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516088. 00000100:00000200:2.0:1773941521.091318:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291481600, offset 0 00000400:00000200:2.0:1773941521.091323:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941521.091330:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941521.091336:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941521.091340:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941521.091343:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941521.091346:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941521.091348:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941521.091350:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[40, 39] 00000400:00000200:2.0:1773941521.091353:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:2.0:1773941521.091356:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [151:151:256:4294967295] 192.168.204.124@tcp LPNI seq info [41:41:8:4294967295] 00000400:00000200:2.0:1773941521.091360:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:2.0:1773941521.091367:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:2.0:1773941521.091371:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:2.0:1773941521.091378:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0db000. 00000800:00000200:2.0:1773941521.091384:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:2.0:1773941521.091390:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:2.0:1773941521.091393:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0db000 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:2.0:1773941521.091406:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941521.091410:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941521.091412:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294912687 00000100:00000001:2.0:1773941521.091418:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294912233 00000100:00000001:2.0:1773941521.091421:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294912687 00000100:00000001:2.0:1773941521.091423:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294912233 00000100:00000001:2.0:1773941521.091426:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941521.091428:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941521.091429:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941521.091431:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:2.0:1773941521.091434:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d09350540 going to sleep for 11 seconds 00000100:00000001:2.0:1773941521.091436:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941521.091438:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941521.091537:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941521.091543:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0db000. 00000400:00000200:0.0:1773941521.091548:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941521.091553:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941521.091557:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516088 00000400:00000010:0.0:1773941521.091558:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516088. 00000100:00000001:0.0:1773941521.091561:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941521.091565:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06506a00 x1860112291481600/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941537 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941521.091574:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941521.091576:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d06506a00 x1860112291481600/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941537 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941521.091581:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941521.091583:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941521.092485:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941521.092496:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941521.092500:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941521.092502:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941521.092506:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941521.092510:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941521.092514:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941521.092517:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941521.092521:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941521.092526:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324604800 00000400:00000200:0.0:1773941521.092530:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941521.092536:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xc51 [1] + 224 00000800:00000001:0.0:1773941521.092543:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941521.092551:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941521.092554:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941521.092558:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941521.092562:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941521.092574:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06506a00 x1860112291481600/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941537 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941521.092586:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d06506a00 x1860112291481600/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941537 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941521.092603:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941521.092609:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941521.092613:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941521.092634:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941521.092639:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941521.092641:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941521.092643:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941521.092654:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941521.092655:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941521.092661:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941521.092665:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06506a00 x1860112291481600/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941537 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941521.092673:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06506a00 x1860112291481600/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941537 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941521.092679:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941521.092680:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941521.092682:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941521.092684:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941521.092686:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941521.092693:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941521.092695:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941521.092699:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=22 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941521.092702:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941521.092704:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:2.0:1773941521.092707:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:2.0:1773941521.092710:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941521.092712:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:2.0:1773941521.092714:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1773941521.092717:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1773941521.092721:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1773941521.092724:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941521.092726:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941521.092727:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941521.092729:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:2.0:1773941521.092731:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d06506a00 x1860112291481600/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941537 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941521.092736:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941521.092737:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941521.092738:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941521.092740:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941521.092742:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:2.0:1773941521.092743:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941521.092744:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941521.092745:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941521.092746:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941521.092747:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941521.092748:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941521.092749:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1773941521.092750:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1773941521.092752:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941521.092755:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 243 00000400:00000200:2.0:1773941521.092758:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941521.092764:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:2.0:1773941521.092768:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941521.092770:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941521.092772:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941521.092774:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:2.0:1773941521.092776:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941521.092778:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941521.092779:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941521.092781:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941521.092782:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941521.092784:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:2.0:1773941521.092786:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941521.092787:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941521.092788:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941521.092790:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:2.0:1773941521.092818:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:2.0:1773941521.092821:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:2.0:1773941521.092823:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941521.092825:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941521.092827:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941521.092829:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941521.092831:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941521.092832:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941521.092835:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06507b80. 00000020:00000040:2.0:1773941521.092837:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:2.0:1773941521.092839:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941521.092841:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941521.092842:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941521.092844:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941521.092845:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941521.092848:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d035a4000. 00000100:00000001:2.0:1773941521.092851:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941521.092855:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d06507b80 x1860112291481728/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941521.092866:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941521.092872:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941521.092876:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000df552709] to pc [ptlrpcd_rcv+-1] req@ffff988d06507b80 x1860112291481728/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941521.092891:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941521.092893:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:2.0:1773941521.092894:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941521.092896:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941521.092899:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941521.092900:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941521.092901:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d09350540 going to sleep for 1 seconds 00000100:00000001:2.0:1773941521.092903:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941521.092905:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941521.092909:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06506a00 x1860112291481600/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941537 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941521.092914:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941521.092916:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941521.092918:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941521.092918:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941521.092922:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941521.092925:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06506a00 x1860112291481600/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941537 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941521.092928:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941521.092932:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00100000:2.0:1773941521.092932:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291481600:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941521.092937:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941521.092939:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06507b80 x1860112291481728/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941521.092940:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941521.092944:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:2.0:1773941521.092947:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941521.092953:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06506a00 x1860112291481600/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941537 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941521.092954:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941521.092957:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941521.092986:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1773941521.092988:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941521.092991:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000010:2.0:1773941521.092997:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d09350540. 02000000:00000001:3.0:1773941521.092998:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941521.093002:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:2.0:1773941521.093005:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00100000:3.0:1773941521.093007:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06507b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291481728:192.168.204.114@tcp:250:ll_cfg_requeue.0 00010000:00000001:2.0:1773941521.093009:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:2.0:1773941521.093014:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000001:3.0:1773941521.093015:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000001:2.0:1773941521.093017:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:3.0:1773941521.093020:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:2.0:1773941521.093020:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089dfa00 refcount=4 02000000:00000001:3.0:1773941521.093024:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941521.093025:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 02000000:00000001:3.0:1773941521.093031:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:2.0:1773941521.093032:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00010000:2.0:1773941521.093037:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e293d lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000010:3.0:1773941521.093038:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03522800. 02000000:00000001:3.0:1773941521.093044:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941521.093047:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000200:3.0:1773941521.093052:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:2.0:1773941521.093055:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e293d lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:3.0:1773941521.093056:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00010000:00000001:2.0:1773941521.093062:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:2.0:1773941521.093067:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e293d lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:3.0:1773941521.093068:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00000100:00000200:3.0:1773941521.093074:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291481728, portal 25 00010000:00000001:2.0:1773941521.093075:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941521.093078:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00000001:2.0:1773941521.093079:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:3.0:1773941521.093081:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196589440 : -113743512962176 : ffff988d06507b80) 00010000:00010000:2.0:1773941521.093083:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e293d lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:3.0:1773941521.093088:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06507b80 x1860112291481728/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941531 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1773941521.093091:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941521.093094:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941521.093099:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:2.0:1773941521.093101:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e293d lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941521.093120:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00010000:2.0:1773941521.093120:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d089dfa00 already unused, calling callback (ffffffffc12b4c20) 00000100:00000040:3.0:1773941521.093123:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:3.0:1773941521.093125:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acdd0. 00000100:00000200:3.0:1773941521.093127:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291481728, offset 0 10000000:00000001:2.0:1773941521.093134:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:3.0:1773941521.093135:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 10000000:00010000:2.0:1773941521.093141:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e293d lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941521.093142:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941521.093148:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941521.093154:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941521.093158:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941521.093162:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:2.0:1773941521.093162:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000400:00000200:3.0:1773941521.093167:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:2.0:1773941521.093167:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:2.0:1773941521.093172:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:3.0:1773941521.093173:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[40, 41] 00000020:00000040:2.0:1773941521.093177:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089dfa00 refcount=5 00000400:00000200:3.0:1773941521.093178:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941521.093183:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [152:152:256:4294967295] 192.168.204.114@tcp LPNI seq info [42:42:8:4294967295] 00000020:00000001:2.0:1773941521.093184:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00000400:00000200:3.0:1773941521.093190:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:2.0:1773941521.093191:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941521.093196:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:3.0:1773941521.093200:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00010000:00010000:2.0:1773941521.093200:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e293d lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941521.093208:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 10000000:00000001:2.0:1773941521.093215:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000800:00000010:3.0:1773941521.093216:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74c00. 10000000:00010000:2.0:1773941521.093218:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e293d lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941521.093225:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 10000000:01000000:2.0:1773941521.093229:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:2.0:1773941521.093232:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:2.0:1773941521.093235:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941521.093236:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00010000:00010000:2.0:1773941521.093239:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000800:00000200:3.0:1773941521.093240:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74c00 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:2.0:1773941521.093242:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:2.0:1773941521.093246:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1773941521.093247:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1773941521.093249:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941521.093250:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941521.093252:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d089dfa00 with handle 0x8e86767622e293d from hash 00010000:00000001:2.0:1773941521.093254:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000100:00000001:3.0:1773941521.093255:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941521.093255:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941521.093258:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941521.093260:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1773941521.093262:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000400:00000200:3.0:1773941521.093265:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:2.0:1773941521.093265:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000100:00000001:3.0:1773941521.093271:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294912689 00010000:00000001:2.0:1773941521.093271:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941521.093277:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294912687 00010000:00000001:2.0:1773941521.093278:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941521.093281:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941521.093285:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294912689 10000000:00000001:2.0:1773941521.093285:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941521.093290:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e293d lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941521.093292:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294912687 00000100:00000001:3.0:1773941521.093298:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941521.093301:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941521.093304:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941521.093305:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941521.093306:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:2.0:1773941521.093307:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:2.0:1773941521.093308:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941521.093312:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:3.0:1773941521.093314:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941521.093314:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000200:0.0:1773941521.093315:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00010000:2.0:1773941521.093318:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e293d lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941521.093319:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:0.0:1773941521.093319:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74c00. 00000400:00000200:0.0:1773941521.093326:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000040:2.0:1773941521.093328:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d071ee100 count: 0 00000100:00000001:3.0:1773941521.093330:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941521.093333:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000400:00000200:0.0:1773941521.093333:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:3.0:1773941521.093337:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:0.0:1773941521.093338:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acdd0 00000100:00000001:3.0:1773941521.093339:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00010000:00000001:2.0:1773941521.093340:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000010:0.0:1773941521.093340:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acdd0. 00000100:00000001:3.0:1773941521.093341:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941521.093345:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941521.093345:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941521.093345:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1773941521.093350:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000200:0.0:1773941521.093352:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06507b80 x1860112291481728/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941531 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941521.093357:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06506a00 x1860112291481600/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941537 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941521.093382:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1773941521.093382:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1773941521.093386:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941521.093389:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a4800. 00000100:00000040:0.0:1773941521.093391:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06507b80 x1860112291481728/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941531 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1773941521.093393:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941521.093397:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941521.093400:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941521.093405:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941521.093408:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941521.093409:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d08a4f400. 00000100:00000001:0.0:1773941521.093415:0:2393:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1773941521.093416:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941521.093418:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941521.093419:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06506a00. 00000100:00000001:2.0:1773941521.093422:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941521.093423:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:2.0:1773941521.093425:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:2.0:1773941521.093430:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:2.0:1773941521.093432:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:2.0:1773941521.093435:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d05437280. 00000020:00000010:2.0:1773941521.093438:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d08a4f400. 00000020:00000010:2.0:1773941521.093441:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d08a4fe00. 00000040:00000010:2.0:1773941521.093445:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d03356200. 00000020:00000010:2.0:1773941521.093449:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941521.093453:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d0381e000. 80000000:00000010:2.0:1773941521.093456:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d08bbb540. 00040000:00000010:2.0:1773941521.093459:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d0381f200. 20000000:00000010:2.0:1773941521.093463:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d0381e800. 10000000:00000040:2.0:1773941521.093466:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:2.0:1773941521.093467:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:2.0:1773941521.093471:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d08a4fe00. 00000040:00000010:2.0:1773941521.093474:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d03356200. 00000020:00000010:2.0:1773941521.093477:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941521.093481:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d0381e000. 80000000:00000010:2.0:1773941521.093483:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d08bbb540. 00040000:00000010:2.0:1773941521.093485:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d0381f200. 20000000:00000010:2.0:1773941521.093487:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d0381e800. 00000020:00000010:2.0:1773941521.093489:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d08a4f400. 10000000:00000010:2.0:1773941521.093491:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d05437280. 10000000:01000000:2.0:1773941521.093493:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:2.0:1773941521.093495:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:2.0:1773941521.093497:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 00000800:00000001:0.0:1773941521.094648:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941521.094654:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941521.094655:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941521.094657:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941521.094660:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941521.094663:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941521.094667:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941521.094670:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941521.094674:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941521.094683:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324604880 00000400:00000200:0.0:1773941521.094687:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941521.094697:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xc59 [1] + 0 00000800:00000001:0.0:1773941521.094704:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941521.094710:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941521.094713:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941521.094716:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941521.094720:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941521.094725:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06507b80 x1860112291481728/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941531 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941521.094735:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d06507b80 x1860112291481728/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941531 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941521.094749:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941521.094754:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941521.094758:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941521.094767:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941521.094769:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941521.094772:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941521.094774:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941521.094776:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941521.094780:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941521.094782:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941521.094784:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941521.094789:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06507b80 x1860112291481728/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941531 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941521.094810:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06507b80 x1860112291481728/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941531 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941521.094814:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941521.094816:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941521.094817:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941521.094818:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941521.094819:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941521.094821:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941521.094822:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941521.094825:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=22 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941521.094827:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941521.094829:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941521.094830:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941521.094832:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941521.094833:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941521.094835:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941521.094838:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06507b80 x1860112291481728/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941531 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941521.094851:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941521.094853:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941521.094854:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941521.094856:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941521.094860:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941521.094862:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941521.094863:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941521.094864:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941521.094866:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941521.094868:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941521.094869:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941521.094872:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941521 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941521.094874:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941521.094878:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941521.094880:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941521.094881:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941521.094882:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941521.094883:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941521.094885:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941521.094887:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941521.094889:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941521.094891:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941521.094893:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941521.094895:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941521.095096:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941521.095098:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941521.095103:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06507b80 x1860112291481728/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941531 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941521.095108:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06507b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291481728:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941521.095118:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941521.095120:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941521.095123:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06507b80 x1860112291481728/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941531 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941521.095128:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941521.095130:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941521.095132:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03522800. 02000000:00000001:3.0:1773941521.095134:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941521.095135:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941521.095136:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941521.095138:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941521.095138:0:14734:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 02000000:00000010:3.0:1773941521.095141:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d035a4000. 02000000:00000001:3.0:1773941521.095144:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941521.095145:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941521.095147:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06507b80. 00000100:00000001:3.0:1773941521.095149:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941521.095151:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1773941521.095152:0:14734:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00000001:3.0:1773941521.095153:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941521.095159:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:1.0:1773941521.095160:0:14734:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:3.0:1773941521.095163:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941521.095165:0:14734:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:3.0:1773941521.095168:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 10000000:01000000:1.0:1773941521.095170:0:14734:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000001:3.0:1773941521.095172:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941521.095176:0:14734:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941521.095180:0:14734:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941521.095184:0:14734:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941521.095188:0:14734:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941521.095189:0:14734:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941521.095192:0:14734:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941521.095194:0:14734:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941521.095196:0:14734:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:1.0:1773941521.095200:0:14734:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:1.0:1773941521.095202:0:14734:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941521.095204:0:14734:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941521.095207:0:14734:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941521.095209:0:14734:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941521.095211:0:14734:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941521.095213:0:14734:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941521.095222:0:14734:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:1.0:1773941521.095235:0:14734:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941521.095237:0:14734:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941521.095238:0:14734:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941521.095242:0:14734:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:1.0:1773941521.095246:0:14734:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941521.095249:0:14734:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941521.095251:0:14734:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941521.095253:0:14734:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941521.095255:0:14734:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941521.099576:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:2.0:1773941521.099582:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:2.0:1773941521.099585:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941521.099590:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00010000:00000010:2.1:1773941521.109001:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d071ee100. 00010000:00000010:2.1:1773941521.109016:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d089dfa00. 00000020:00000001:1.0:1773941521.403127:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1773941521.403140:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941521.403164:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1773941522.363113:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294912689, at 4294912704 00000100:00000010:0.0:1773941522.363128:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f34e0. 00000100:00000040:1.0:1773941522.363146:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294912689, at 4294912704 00000100:00000010:1.0:1773941522.363156:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3300. 00000020:00000001:2.0:1773941522.427126:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941522.427135:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941522.427140:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:2.0:1773941522.491022:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:2.0:1773941522.491030:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941522.491056:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941522.491059:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941522.491060:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941522.491061:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:2.0:1773941522.491065:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:2.0:1773941522.491066:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:2.0:1773941522.491069:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:2.0:1773941522.491071:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:2.0:1773941522.491073:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:2.0:1773941522.491080:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06506a00. 00000020:00000040:2.0:1773941522.491085:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941522.491090:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941522.491094:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941522.491100:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941522.491107:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d0381e800. 00000100:00000001:2.0:1773941522.491115:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941522.491120:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:2.0:1773941522.491130:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:2.0:1773941522.491138:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d071ee100. 00010000:00000001:2.0:1773941522.491145:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:2.0:1773941522.491148:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d089dfa00. 00000020:00000001:2.0:1773941522.491153:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:2.0:1773941522.491155:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d089dfa00 with handle 0x8e86767622e2944 to hash 00000020:00000001:2.0:1773941522.491157:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:2.0:1773941522.491159:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941522.491161:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941522.491165:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941522.491168:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941522.491171:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2944 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941522.491180:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2944 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941522.491191:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2944 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941522.491200:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:2.0:1773941522.491202:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:2.0:1773941522.491209:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d09350540. 00000100:00000001:2.0:1773941522.491210:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330245121344 : -113743464430272 : ffff988d09350540) 00000100:00000001:2.0:1773941522.491213:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941522.491214:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584960 : -113743512966656 : ffff988d06506a00) 00000100:00000001:2.0:1773941522.491219:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941522.491224:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941522.491226:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:2.0:1773941522.491228:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941522.491238:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06506a00 x1860112291481856/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941522.491245:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941522.491247:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941522.491257:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941522.491259:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941522.491261:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941522.491264:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291481856:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941522.491268:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941522.491274:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941522.491276:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941522.491279:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941522.491285:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a4400. 02000000:00000001:2.0:1773941522.491287:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941522.491297:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941522.491302:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000400:00000010:2.0:1773941522.491312:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000200:2.0:1773941522.491320:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291481856, portal 25 00000100:00000001:2.0:1773941522.491322:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941522.491324:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584960 : -113743512966656 : ffff988d06506a00) 00000100:00000040:2.0:1773941522.491330:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06506a00 x1860112291481856/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941538 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941522.491353:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941522.491357:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:2.0:1773941522.491360:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516198. 00000100:00000200:2.0:1773941522.491365:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291481856, offset 0 00000400:00000200:2.0:1773941522.491373:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941522.491385:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941522.491395:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941522.491402:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941522.491407:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941522.491412:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941522.491416:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941522.491420:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[42, 41] 00000400:00000200:2.0:1773941522.491424:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:2.0:1773941522.491429:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [153:153:256:4294967295] 192.168.204.124@tcp LPNI seq info [43:43:8:4294967295] 00000400:00000200:2.0:1773941522.491434:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:2.0:1773941522.491448:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:2.0:1773941522.491456:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:2.0:1773941522.491462:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0db900. 00000800:00000200:2.0:1773941522.491470:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:2.0:1773941522.491477:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:2.0:1773941522.491481:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0db900 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:2.0:1773941522.491497:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941522.491501:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941522.491504:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294914087 00000100:00000010:2.0:1773941522.491507:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d08bbb540. 00000100:00000001:2.0:1773941522.491514:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1773941522.491521:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294914087 00000100:00000010:2.0:1773941522.491523:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d08bbbc60. 00000100:00000001:2.0:1773941522.491526:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1773941522.491534:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941522.491536:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941522.491538:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941522.491540:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:2.0:1773941522.491542:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d09350540 going to sleep for 11 seconds 00000100:00000001:2.0:1773941522.491546:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941522.491549:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941522.491686:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941522.491694:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0db900. 00000400:00000200:0.0:1773941522.491700:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941522.491708:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941522.491715:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516198 00000400:00000010:0.0:1773941522.491717:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516198. 00000100:00000001:0.0:1773941522.491722:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941522.491729:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06506a00 x1860112291481856/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941538 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941522.491738:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941522.491741:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d06506a00 x1860112291481856/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941538 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941522.491747:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941522.491748:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941522.492947:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941522.492995:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941522.492997:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941522.493011:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941522.493017:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941522.493021:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941522.493026:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941522.493029:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941522.493033:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941522.493036:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324604900 00000400:00000200:0.0:1773941522.493039:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941522.493044:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xc61 [1] + 224 00000800:00000001:0.0:1773941522.493049:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941522.493056:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941522.493058:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941522.493061:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941522.493064:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941522.493067:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06506a00 x1860112291481856/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941538 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941522.493076:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d06506a00 x1860112291481856/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941538 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941522.493093:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941522.493099:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941522.493101:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941522.493131:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941522.493139:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941522.493142:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941522.493145:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941522.493151:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941522.493153:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941522.493157:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941522.493164:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06506a00 x1860112291481856/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941538 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941522.493177:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06506a00 x1860112291481856/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941538 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941522.493185:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941522.493187:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941522.493190:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941522.493193:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941522.493196:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941522.493199:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941522.493203:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941522.493210:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=23 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941522.493215:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941522.493219:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:2.0:1773941522.493223:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:2.0:1773941522.493227:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941522.493231:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:2.0:1773941522.493234:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1773941522.493240:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1773941522.493245:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1773941522.493263:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941522.493265:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941522.493277:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941522.493279:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:2.0:1773941522.493283:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d06506a00 x1860112291481856/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941538 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941522.493289:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941522.493290:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941522.493292:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941522.493294:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941522.493297:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:2.0:1773941522.493298:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941522.493300:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941522.493300:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941522.493301:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941522.493302:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941522.493303:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941522.493306:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1773941522.493307:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1773941522.493309:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941522.493311:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 244 00000400:00000200:2.0:1773941522.493316:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941522.493320:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:2.0:1773941522.493323:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941522.493326:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941522.493328:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941522.493330:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:2.0:1773941522.493343:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941522.493344:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941522.493346:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941522.493348:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941522.493349:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941522.493351:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:2.0:1773941522.493359:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941522.493361:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941522.493363:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941522.493364:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:2.0:1773941522.493367:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:2.0:1773941522.493369:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:2.0:1773941522.493371:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941522.493373:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941522.493375:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941522.493377:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941522.493379:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941522.493380:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941522.493383:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06505880. 00000020:00000040:2.0:1773941522.493385:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:2.0:1773941522.493387:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941522.493389:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941522.493391:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941522.493392:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941522.493394:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941522.493398:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d035a7800. 00000100:00000001:2.0:1773941522.493401:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941522.493405:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d06505880 x1860112291481984/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941522.493411:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941522.493413:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941522.493417:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e62ba1de] to pc [ptlrpcd_rcv+-1] req@ffff988d06505880 x1860112291481984/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941522.493436:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941522.493438:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:2.0:1773941522.493440:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941522.493442:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941522.493444:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941522.493449:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941522.493451:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d09350540 going to sleep for 1 seconds 00000100:00000001:2.0:1773941522.493452:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941522.493454:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941522.493459:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06506a00 x1860112291481856/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941538 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941522.493464:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941522.493467:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941522.493469:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941522.493471:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941522.493475:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06506a00 x1860112291481856/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941538 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941522.493476:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941522.493483:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1773941522.493484:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291481856:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941522.493489:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1773941522.493491:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941522.493494:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941522.493496:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:2.0:1773941522.493498:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941522.493501:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06505880 x1860112291481984/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941522.493503:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06506a00 x1860112291481856/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941538 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941522.493533:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941522.493536:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000010:2.0:1773941522.493537:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d09350540. 00000100:00000001:3.0:1773941522.493540:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941522.493541:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:2.0:1773941522.493543:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:3.0:1773941522.493547:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00010000:00000001:2.0:1773941522.493548:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 02000000:00000001:3.0:1773941522.493552:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00010000:00000001:2.0:1773941522.493554:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:2.0:1773941522.493557:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:3.0:1773941522.493558:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941522.493561:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089dfa00 refcount=4 00000100:00100000:3.0:1773941522.493565:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06505880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291481984:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000020:00000001:2.0:1773941522.493566:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941522.493571:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00000100:00000001:3.0:1773941522.493573:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00010000:2.0:1773941522.493576:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2944 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941522.493584:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941522.493587:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941522.493594:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 02000000:00000001:3.0:1773941522.493597:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941522.493603:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03520000. 00010000:00010000:2.0:1773941522.493604:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2944 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941522.493607:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941522.493616:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:2.0:1773941522.493617:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000400:00000010:3.0:1773941522.493620:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00010000:00010000:2.0:1773941522.493621:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2944 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:3.0:1773941522.493627:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00010000:00000001:2.0:1773941522.493629:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941522.493633:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000200:3.0:1773941522.493634:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291481984, portal 25 00010000:00010000:2.0:1773941522.493635:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2944 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941522.493638:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941522.493641:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196580480 : -113743512971136 : ffff988d06505880) 00010000:00000001:2.0:1773941522.493648:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000040:3.0:1773941522.493650:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06505880 x1860112291481984/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941532 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1773941522.493651:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941522.493656:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:2.0:1773941522.493660:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2944 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941522.493664:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941522.493669:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00010000:2.0:1773941522.493670:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d089dfa00 already unused, calling callback (ffffffffc12b4c20) 00000400:00000010:3.0:1773941522.493672:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac660. 10000000:00000001:2.0:1773941522.493673:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000200:3.0:1773941522.493678:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291481984, offset 0 10000000:00010000:2.0:1773941522.493678:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2944 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941522.493689:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00000001:2.0:1773941522.493690:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:2.0:1773941522.493693:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:2.0:1773941522.493694:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941522.493704:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089dfa00 refcount=5 00000400:00000200:3.0:1773941522.493707:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000020:00000001:2.0:1773941522.493708:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941522.493712:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00000400:00000200:3.0:1773941522.493713:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:2.0:1773941522.493716:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:3.0:1773941522.493718:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941522.493722:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00010000:2.0:1773941522.493724:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2944 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941522.493726:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941522.493731:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941522.493737:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[42, 43] 10000000:00000001:2.0:1773941522.493741:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:3.0:1773941522.493744:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 10000000:00010000:2.0:1773941522.493746:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2944 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941522.493750:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [154:154:256:4294967295] 192.168.204.114@tcp LPNI seq info [44:44:8:4294967295] 10000000:01000000:2.0:1773941522.493761:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:3.0:1773941522.493762:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 10000000:00000040:2.0:1773941522.493776:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:2.0:1773941522.493778:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941522.493781:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:2.0:1773941522.493785:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:3.0:1773941522.493788:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00010000:00000001:2.0:1773941522.493789:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1773941522.493793:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000800:00000200:3.0:1773941522.493797:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00010000:00000001:2.0:1773941522.493797:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941522.493801:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1773941522.493803:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74800. 00000020:00000040:2.0:1773941522.493804:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d089dfa00 with handle 0x8e86767622e2944 from hash 00010000:00000001:2.0:1773941522.493818:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1773941522.493819:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941522.493820:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941522.493822:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000800:00000200:3.0:1773941522.493824:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00010000:00000001:2.0:1773941522.493825:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:2.0:1773941522.493827:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:2.0:1773941522.493828:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941522.493829:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941522.493830:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941522.493831:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941522.493836:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2944 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941522.493842:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000200:3.0:1773941522.493843:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00010000:00000001:2.0:1773941522.493844:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941522.493847:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000800:00000200:3.0:1773941522.493849:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74800 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:2.0:1773941522.493850:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:2.0:1773941522.493853:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941522.493854:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:3.0:1773941522.493862:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941522.493863:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1773941522.493867:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2944 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941522.493868:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:3.0:1773941522.493874:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294914089 00010000:00000040:2.0:1773941522.493881:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d071ee100 count: 0 00000100:00000001:3.0:1773941522.493884:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294914087 00000100:00000001:3.0:1773941522.493895:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294914089 00000100:00000001:3.0:1773941522.493898:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294914087 00010000:00000001:2.0:1773941522.493902:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:3.0:1773941522.493903:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:2.0:1773941522.493907:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941522.493908:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941522.493912:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1773941522.493913:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941522.493919:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941522.493920:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06506a00 x1860112291481856/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941538 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941522.493930:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941522.493931:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1773941522.493934:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 02000000:00000001:2.0:1773941522.493935:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1773941522.493937:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941522.493940:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 02000000:00000010:2.0:1773941522.493940:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a4400. 00000100:00000001:3.0:1773941522.493944:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941522.493945:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 02000000:00000001:2.0:1773941522.493946:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1773941522.493947:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941522.493951:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000800:00000010:0.0:1773941522.493960:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74800. 00000400:00000200:0.0:1773941522.494010:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000040:2.0:1773941522.494016:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941522.494022:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 00000400:00000200:0.0:1773941522.494023:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 02000000:00000010:2.0:1773941522.494034:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d0381e800. 00000400:00000200:0.0:1773941522.494036:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac660 00000400:00000010:0.0:1773941522.494041:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac660. 02000000:00000001:2.0:1773941522.494042:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941522.494045:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1773941522.494048:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000010:2.0:1773941522.494049:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06506a00. 00000100:00000001:2.0:1773941522.494055:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000200:0.0:1773941522.494058:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06505880 x1860112291481984/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941532 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941522.494059:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:2.0:1773941522.494070:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:2.0:1773941522.494081:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:2.0:1773941522.494088:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000100:00000001:0.0:1773941522.494092:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 10000000:00000010:2.0:1773941522.494094:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d05437280. 00000020:00000010:2.0:1773941522.494100:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d0381e800. 00000100:00000040:0.0:1773941522.494102:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06505880 x1860112291481984/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941532 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1773941522.494106:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d0381f200. 00000040:00000010:2.0:1773941522.494112:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d0381e000. 00000020:00000010:2.0:1773941522.494118:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941522.494125:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d03356200. 80000000:00000010:2.0:1773941522.494133:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d08bbbae0. 00000100:00000001:0.0:1773941522.494133:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00040000:00000010:2.0:1773941522.494138:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d08a4f400. 00000100:00000001:0.0:1773941522.494139:0:2393:0:(events.c:68:request_out_callback()) Process leaving 20000000:00000010:2.0:1773941522.494148:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d08a4fe00. 10000000:00000040:2.0:1773941522.494159:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:2.0:1773941522.494161:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:2.0:1773941522.494167:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d0381f200. 00000040:00000010:2.0:1773941522.494171:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d0381e000. 00000020:00000010:2.0:1773941522.494175:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941522.494178:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d03356200. 80000000:00000010:2.0:1773941522.494181:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d08bbbae0. 00040000:00000010:2.0:1773941522.494184:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d08a4f400. 20000000:00000010:2.0:1773941522.494187:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d08a4fe00. 00000020:00000010:2.0:1773941522.494190:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0381e800. 10000000:00000010:2.0:1773941522.494192:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d05437280. 10000000:01000000:2.0:1773941522.494194:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:2.0:1773941522.494196:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:2.0:1773941522.494198:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:2.0:1773941522.494200:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:2.0:1773941522.494200:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:2.0:1773941522.494202:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941522.494203:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941522.495315:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941522.495322:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941522.495324:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941522.495326:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941522.495330:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941522.495334:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941522.495338:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941522.495340:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941522.495343:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941522.495345:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324604980 00000400:00000200:0.0:1773941522.495349:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941522.495353:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xc69 [1] + 0 00000800:00000001:0.0:1773941522.495358:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941522.495363:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941522.495364:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941522.495367:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941522.495370:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941522.495373:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06505880 x1860112291481984/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941532 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941522.495380:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d06505880 x1860112291481984/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941532 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941522.495394:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941522.495397:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941522.495399:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941522.495425:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941522.495428:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941522.495432:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941522.495433:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941522.495435:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941522.495440:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941522.495441:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941522.495444:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941522.495449:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06505880 x1860112291481984/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941532 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941522.495458:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06505880 x1860112291481984/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941532 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941522.495464:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941522.495465:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941522.495466:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941522.495474:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941522.495476:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941522.495477:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941522.495479:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941522.495482:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=23 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941522.495485:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941522.495487:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941522.495488:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941522.495490:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941522.495492:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941522.495494:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941522.495497:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06505880 x1860112291481984/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941532 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941522.495502:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941522.495503:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941522.495505:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941522.495507:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941522.495521:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941522.495522:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941522.495523:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941522.495525:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941522.495527:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941522.495528:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941522.495530:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941522.495533:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941522 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941522.495535:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941522.495540:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941522.495541:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941522.495543:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941522.495544:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941522.495545:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941522.495548:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941522.495550:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941522.495552:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941522.495558:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941522.495561:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941522.495563:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941522.495797:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941522.495801:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941522.495807:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06505880 x1860112291481984/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941532 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941522.495828:0:14735:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:2.0:1773941522.495828:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06505880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291481984:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941522.495835:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941522.495838:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1773941522.495840:0:14735:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00000040:2.0:1773941522.495844:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06505880 x1860112291481984/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941532 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941522.495846:0:14735:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:3.0:1773941522.495850:0:14735:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941522.495853:0:14735:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:3.0:1773941522.495858:0:14735:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:2.0:1773941522.495858:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:3.0:1773941522.495861:0:14735:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941522.495862:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941522.495866:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03520000. 00000001:00000001:3.0:1773941522.495867:0:14735:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941522.495869:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:3.0:1773941522.495870:0:14735:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941522.495872:0:14735:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941522.495874:0:14735:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941522.495875:0:14735:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941522.495876:0:14735:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000001:2.0:1773941522.495880:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000100:00080000:3.0:1773941522.495881:0:14735:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000020:00000040:2.0:1773941522.495882:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941522.495887:0:14735:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000020:00000001:2.0:1773941522.495888:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941522.495891:0:14735:0:(recover.c:147:ptlrpc_resend()) Process entered 02000000:00000010:2.0:1773941522.495891:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d035a7800. 00000100:00000001:3.0:1773941522.495894:0:14735:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941522.495896:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00080000:3.0:1773941522.495897:0:14735:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 02000000:00000001:2.0:1773941522.495900:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1773941522.495901:0:14735:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000010:2.0:1773941522.495903:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06505880. 10000000:01000000:3.0:1773941522.495904:0:14735:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:3.0:1773941522.495908:0:14735:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00000001:2.0:1773941522.495909:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941522.495912:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941522.495914:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941522.495917:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941522.495918:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941522.495920:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941522.495921:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1773941522.495924:0:14735:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:3.0:1773941522.495927:0:14735:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941522.495928:0:14735:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:3.0:1773941522.495932:0:14735:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:3.0:1773941522.495934:0:14735:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941522.495936:0:14735:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941522.495938:0:14735:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941522.495940:0:14735:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941522.495941:0:14735:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:2.1:1773941522.498998:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d071ee100. 00010000:00000010:2.1:1773941522.499004:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d089dfa00. 00000020:00000001:1.0:1773941523.451095:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1773941523.451100:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941523.451104:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:0.0:1773941523.643050:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1773941523.643056:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941523.643060:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941523.643062:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941523.643063:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941523.643064:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1773941523.643094:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:0.0:1773941523.643096:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:0.0:1773941523.643098:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:0.0:1773941523.643100:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1773941523.643101:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:0.0:1773941523.643107:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10835880. 00000020:00000040:0.0:1773941523.643112:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941523.643115:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941523.643118:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941523.643123:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941523.643129:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065c4c00. 00000100:00000001:0.0:1773941523.643134:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941523.643138:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1773941523.643140:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1773941523.643146:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d21ec1800. 00010000:00000001:0.0:1773941523.643156:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1773941523.643158:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d00030000. 00000020:00000001:0.0:1773941523.643161:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:0.0:1773941523.643163:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d00030000 with handle 0x8e86767622e294b to hash 00000020:00000001:0.0:1773941523.643164:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:0.0:1773941523.643166:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:0.0:1773941523.643168:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00010000:00000001:0.0:1773941523.643171:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941523.643173:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941523.643177:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00030000/0x8e86767622e294b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941523.643185:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d00030000/0x8e86767622e294b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941523.643196:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d00030000/0x8e86767622e294b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941523.643203:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1773941523.643213:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941523.643219:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb6840. 00000100:00000001:0.0:1773941523.643221:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330139633728 : -113743569917888 : ffff988d02eb6840) 00000100:00000001:0.0:1773941523.643224:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941523.643226:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367694976 : -113743341856640 : ffff988d10835880) 00000100:00000001:0.0:1773941523.643231:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941523.643236:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941523.643238:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941523.643240:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941523.643252:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10835880 x1860112291482112/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941523.643262:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941523.643264:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941523.643268:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941523.643271:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941523.643274:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941523.643278:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10835880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291482112:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941523.643283:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941523.643288:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941523.643290:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941523.643293:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941523.643297:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d02ce5800. 02000000:00000001:0.0:1773941523.643300:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941523.643307:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941523.643312:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0014b7e8. 00000400:00000010:0.0:1773941523.643318:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833330. 00000100:00000200:0.0:1773941523.643325:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291482112, portal 25 00000100:00000001:0.0:1773941523.643326:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941523.643327:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367694976 : -113743341856640 : ffff988d10835880) 00000100:00000040:0.0:1773941523.643331:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10835880 x1860112291482112/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941539 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941523.643337:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941523.643339:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941523.643345:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833cc0. 00000100:00000200:0.0:1773941523.643348:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291482112, offset 0 00000400:00000200:0.0:1773941523.643353:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941523.643359:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941523.643365:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941523.643370:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941523.643373:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941523.643377:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941523.643379:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941523.643382:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[44, 43] 00000400:00000200:0.0:1773941523.643384:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:0.0:1773941523.643387:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [155:155:256:4294967295] 192.168.204.124@tcp LPNI seq info [45:45:8:4294967295] 00000400:00000200:0.0:1773941523.643390:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941523.643397:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:0.0:1773941523.643402:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:0.0:1773941523.643407:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a20bd00. 00000800:00000200:0.0:1773941523.643412:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:0.0:1773941523.643418:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:0.0:1773941523.643422:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a20bd00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1773941523.643438:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941523.643442:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941523.643444:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294915239 00000100:00000001:0.0:1773941523.643450:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294914089 00000100:00000001:0.0:1773941523.643454:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294915239 00000100:00000001:0.0:1773941523.643457:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294914089 00000100:00000001:0.0:1773941523.643459:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941523.643461:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941523.643480:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941523.643482:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1773941523.643485:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6840 going to sleep for 11 seconds 00000100:00000001:0.0:1773941523.643493:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941523.643497:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941523.643674:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941523.643680:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a20bd00. 00000400:00000200:1.0:1773941523.643685:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941523.643691:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941523.643695:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833cc0 00000400:00000010:1.0:1773941523.643698:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833cc0. 00000100:00000001:1.0:1773941523.643701:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941523.643705:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10835880 x1860112291482112/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941539 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941523.643715:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941523.643717:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d10835880 x1860112291482112/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941539 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941523.643723:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941523.643725:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941523.644948:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941523.644975:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941523.644977:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941523.644980:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941523.644985:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941523.644989:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941523.645011:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941523.645014:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941523.645017:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941523.645020:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324604a00 00000400:00000200:0.0:1773941523.645023:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941523.645027:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xc71 [1] + 224 00000800:00000001:0.0:1773941523.645032:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941523.645039:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941523.645041:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941523.645044:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941523.645047:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941523.645050:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10835880 x1860112291482112/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941539 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941523.645059:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d10835880 x1860112291482112/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941539 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941523.645075:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941523.645082:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941523.645085:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941523.645147:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941523.645162:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941523.645165:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941523.645169:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0014b7e8. 00000400:00000200:2.0:1773941523.645179:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833330 00000400:00000010:2.0:1773941523.645182:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833330. 00000100:00000001:2.0:1773941523.645186:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941523.645197:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10835880 x1860112291482112/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941539 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941523.645212:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10835880 x1860112291482112/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941539 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941523.645222:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941523.645225:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941523.645228:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941523.645232:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941523.645235:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941523.645247:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941523.645250:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941523.645259:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=24 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941523.645264:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941523.645268:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:2.0:1773941523.645273:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:2.0:1773941523.645276:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941523.645297:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:2.0:1773941523.645300:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1773941523.645315:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1773941523.645320:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1773941523.645325:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941523.645328:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941523.645329:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941523.645332:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:2.0:1773941523.645337:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d10835880 x1860112291482112/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941539 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941523.645346:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941523.645348:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941523.645350:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941523.645354:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941523.645357:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:2.0:1773941523.645359:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941523.645362:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941523.645363:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941523.645364:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941523.645366:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941523.645367:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941523.645370:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1773941523.645372:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1773941523.645375:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941523.645377:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 245 00000400:00000200:2.0:1773941523.645384:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941523.645390:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:2.0:1773941523.645395:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941523.645399:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941523.645402:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941523.645404:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:2.0:1773941523.645409:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941523.645417:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941523.645420:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941523.645423:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941523.645426:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941523.645429:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:2.0:1773941523.645432:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941523.645435:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941523.645436:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941523.645438:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:2.0:1773941523.645442:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:2.0:1773941523.645445:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:2.0:1773941523.645448:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941523.645451:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941523.645455:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941523.645458:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941523.645460:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941523.645463:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941523.645469:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06505880. 00000020:00000040:2.0:1773941523.645472:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:2.0:1773941523.645477:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941523.645479:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941523.645482:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941523.645484:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941523.645488:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941523.645495:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d035a7800. 00000100:00000001:2.0:1773941523.645501:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941523.645507:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d06505880 x1860112291482240/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941523.645517:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941523.645521:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941523.645526:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e62ba1de] to pc [ptlrpcd_rcv+-1] req@ffff988d06505880 x1860112291482240/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941523.645567:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941523.645570:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:2.0:1773941523.645572:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941523.645576:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941523.645579:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941523.645581:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941523.645584:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6840 going to sleep for 1 seconds 00000100:00000001:2.0:1773941523.645587:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941523.645589:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941523.645597:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10835880 x1860112291482112/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941539 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941523.645606:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941523.645609:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941523.645610:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941523.645614:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941523.645615:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941523.645618:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941523.645624:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10835880 x1860112291482112/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941539 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941523.645626:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941523.645634:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06505880 x1860112291482240/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941523.645643:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10835880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291482112:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941523.645649:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941523.645652:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941523.645655:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941523.645657:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941523.645660:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941523.645660:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941523.645667:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00100000:3.0:1773941523.645668:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06505880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291482240:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941523.645674:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:3.0:1773941523.645675:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1773941523.645677:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1773941523.645679:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941523.645682:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941523.645685:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10835880 x1860112291482112/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941539 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1773941523.645686:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941523.645691:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03523800. 02000000:00000001:3.0:1773941523.645695:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941523.645702:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941523.645704:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941523.645709:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00000100:00000010:2.0:1773941523.645709:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb6840. 00000100:00000001:2.0:1773941523.645717:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000400:00000010:3.0:1773941523.645718:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00000100:00000001:2.0:1773941523.645721:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000200:3.0:1773941523.645724:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291482240, portal 25 00000100:00000001:3.0:1773941523.645728:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00000001:2.0:1773941523.645728:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000001:3.0:1773941523.645730:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196580480 : -113743512971136 : ffff988d06505880) 00000100:00000040:3.0:1773941523.645735:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06505880 x1860112291482240/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941533 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1773941523.645740:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000001:3.0:1773941523.645743:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:2.0:1773941523.645760:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941523.645763:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030000 refcount=4 00000020:00000001:2.0:1773941523.645766:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00000100:00000040:3.0:1773941523.645772:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00000001:2.0:1773941523.645773:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00000400:00000010:3.0:1773941523.645776:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac7f8. 00010000:00010000:2.0:1773941523.645780:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d00030000/0x8e86767622e294b lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:3.0:1773941523.645781:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291482240, offset 0 00000400:00000200:3.0:1773941523.645791:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00000001:2.0:1773941523.645797:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000200:3.0:1773941523.645800:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:2.0:1773941523.645807:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d00030000/0x8e86767622e294b lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941523.645808:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941523.645816:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:2.0:1773941523.645832:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000400:00000200:3.0:1773941523.645834:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00010000:2.0:1773941523.645837:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00030000/0x8e86767622e294b lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941523.645838:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941523.645849:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:2.0:1773941523.645850:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:3.0:1773941523.645854:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[44, 45] 00010000:00000001:2.0:1773941523.645855:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:3.0:1773941523.645860:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00010000:00010000:2.0:1773941523.645861:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d00030000/0x8e86767622e294b lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941523.645867:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [156:156:256:4294967295] 192.168.204.114@tcp LPNI seq info [46:46:8:4294967295] 00000400:00000200:3.0:1773941523.645878:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:2.0:1773941523.645879:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941523.645883:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941523.645887:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:3.0:1773941523.645890:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00010000:00010000:2.0:1773941523.645893:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d00030000/0x8e86767622e294b lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941523.645899:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:3.0:1773941523.645906:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74f00. 00010000:00010000:2.0:1773941523.645908:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d00030000 already unused, calling callback (ffffffffc12b4c20) 10000000:00000001:2.0:1773941523.645912:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000800:00000200:3.0:1773941523.645915:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 10000000:00010000:2.0:1773941523.645918:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d00030000/0x8e86767622e294b lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941523.645926:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00010000:00000001:2.0:1773941523.645930:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000800:00000200:3.0:1773941523.645931:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74f00 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:2.0:1773941523.645944:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000001:3.0:1773941523.645946:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941523.645946:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941523.645951:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030000 refcount=5 00000400:00000200:3.0:1773941523.645954:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000020:00000001:2.0:1773941523.645957:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00000100:00000001:3.0:1773941523.645958:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294915241 00000100:00000001:3.0:1773941523.646008:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294915239 00000100:00000001:3.0:1773941523.646013:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294915242 00010000:00000001:2.0:1773941523.646018:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090848256 : -113743618703360 : ffff988d00030000) 00000100:00000001:3.0:1773941523.646021:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294915239 00010000:00000001:2.0:1773941523.646025:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000100:00000001:3.0:1773941523.646027:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941523.646031:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941523.646032:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d00030000/0x8e86767622e294b lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941523.646036:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941523.646042:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:2.0:1773941523.646048:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000800:00000200:1.0:1773941523.646050:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1773941523.646052:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 10000000:00010000:2.0:1773941523.646053:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d00030000/0x8e86767622e294b lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941523.646057:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000800:00000010:1.0:1773941523.646058:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74f00. 00000100:00000001:3.0:1773941523.646077:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 10000000:01000000:2.0:1773941523.646077:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:1.0:1773941523.646077:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1773941523.646087:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 10000000:00000040:2.0:1773941523.646091:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000100:00000001:3.0:1773941523.646095:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941523.646095:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 10000000:00000001:2.0:1773941523.646097:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941523.646106:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941523.646113:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac7f8 00010000:00010000:2.0:1773941523.646115:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000400:00000010:1.0:1773941523.646126:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac7f8. 00010000:00000001:2.0:1773941523.646127:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:2.0:1773941523.646136:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000100:00000001:1.0:1773941523.646137:0:2393:0:(events.c:34:request_out_callback()) Process entered 00010000:00000001:2.0:1773941523.646146:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000100:00000200:1.0:1773941523.646151:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06505880 x1860112291482240/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941533 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1773941523.646153:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941523.646160:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941523.646173:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d00030000 with handle 0x8e86767622e294b from hash 00000100:00000001:1.0:1773941523.646173:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00010000:00000001:2.0:1773941523.646185:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000100:00000040:1.0:1773941523.646186:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06505880 x1860112291482240/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941533 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1773941523.646192:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:1.0:1773941523.646197:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941523.646198:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941523.646203:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1773941523.646205:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000100:00000001:1.0:1773941523.646208:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:2.0:1773941523.646210:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:2.0:1773941523.646216:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941523.646218:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941523.646220:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941523.646223:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941523.646228:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d00030000/0x8e86767622e294b lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941523.646247:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941523.646248:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941523.646250:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:2.0:1773941523.646252:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:2.0:1773941523.646254:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941523.646255:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941523.646257:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1773941523.646259:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d00030000/0x8e86767622e294b lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:2.0:1773941523.646268:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d21ec1800 count: 0 00010000:00000001:2.0:1773941523.646284:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941523.646286:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941523.646290:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941523.646297:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10835880 x1860112291482112/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941539 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941523.646306:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941523.646308:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941523.646311:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d02ce5800. 02000000:00000001:2.0:1773941523.646316:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941523.646318:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941523.646320:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941523.646323:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941523.646326:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065c4c00. 02000000:00000001:2.0:1773941523.646331:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941523.646333:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941523.646335:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10835880. 00000100:00000001:2.0:1773941523.646339:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941523.646341:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:2.0:1773941523.646343:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:2.0:1773941523.646346:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:2.0:1773941523.646349:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:2.0:1773941523.646354:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d05437280. 00000020:00000010:2.0:1773941523.646359:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d08a4fe00. 00000020:00000010:2.0:1773941523.646369:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d08a4f400. 00000040:00000010:2.0:1773941523.646375:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d03356200. 00000020:00000010:2.0:1773941523.646382:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941523.646387:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d0381e800. 80000000:00000010:2.0:1773941523.646392:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d08bbbae0. 00040000:00000010:2.0:1773941523.646398:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d0381e000. 20000000:00000010:2.0:1773941523.646404:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d0381f200. 10000000:00000040:2.0:1773941523.646410:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:2.0:1773941523.646413:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:2.0:1773941523.646418:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d08a4f400. 00000040:00000010:2.0:1773941523.646425:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d03356200. 00000020:00000010:2.0:1773941523.646429:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941523.646434:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d0381e800. 80000000:00000010:2.0:1773941523.646437:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d08bbbae0. 00040000:00000010:2.0:1773941523.646441:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d0381e000. 20000000:00000010:2.0:1773941523.646445:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d0381f200. 00000020:00000010:2.0:1773941523.646448:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d08a4fe00. 10000000:00000010:2.0:1773941523.646451:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d05437280. 10000000:01000000:2.0:1773941523.646454:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:2.0:1773941523.646458:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:2.0:1773941523.646461:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:2.0:1773941523.646463:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:2.0:1773941523.646465:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:2.0:1773941523.646467:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941523.646470:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941523.647813:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941523.647823:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941523.647825:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941523.647827:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941523.647833:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941523.647837:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941523.647842:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941523.647852:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941523.647855:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941523.647860:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324604a80 00000400:00000200:0.0:1773941523.647865:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941523.647870:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xc81 [1] + 0 00000800:00000001:0.0:1773941523.647892:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941523.647898:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941523.647900:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941523.647903:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941523.647907:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941523.647912:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06505880 x1860112291482240/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941533 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941523.647922:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d06505880 x1860112291482240/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941533 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941523.647939:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941523.647943:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941523.647946:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941523.647989:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941523.647992:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941523.647998:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941523.648000:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941523.648003:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941523.648014:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941523.648016:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941523.648018:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941523.648024:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06505880 x1860112291482240/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941533 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941523.648033:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06505880 x1860112291482240/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941533 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941523.648038:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941523.648039:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941523.648041:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941523.648042:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941523.648043:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941523.648045:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941523.648046:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941523.648049:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=24 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941523.648051:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941523.648052:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941523.648054:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941523.648056:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941523.648058:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941523.648061:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941523.648064:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06505880 x1860112291482240/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941533 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941523.648068:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941523.648070:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941523.648071:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941523.648072:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941523.648077:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941523.648079:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941523.648080:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941523.648081:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941523.648083:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941523.648085:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941523.648087:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941523.648096:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941523 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941523.648098:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941523.648104:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941523.648106:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941523.648108:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941523.648109:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941523.648111:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941523.648114:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941523.648116:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941523.648118:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941523.648121:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941523.648124:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941523.648126:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941523.648386:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941523.648391:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941523.648397:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06505880 x1860112291482240/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941533 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941523.648400:0:14736:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:2.0:1773941523.648408:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06505880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291482240:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00080000:3.0:1773941523.648411:0:14736:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00000001:2.0:1773941523.648415:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1773941523.648416:0:14736:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:3.0:1773941523.648418:0:14736:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941523.648419:0:14736:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:3.0:1773941523.648422:0:14736:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941523.648427:0:14736:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941523.648427:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000001:00000001:3.0:1773941523.648432:0:14736:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941523.648435:0:14736:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000040:2.0:1773941523.648436:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06505880 x1860112291482240/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941533 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000001:00000001:3.0:1773941523.648437:0:14736:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941523.648439:0:14736:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941523.648443:0:14736:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941523.648447:0:14736:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941523.648452:0:14736:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941523.648455:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1773941523.648456:0:14736:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 02000000:00000001:2.0:1773941523.648459:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1773941523.648460:0:14736:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941523.648464:0:14736:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941523.648465:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03523800. 00000100:00080000:3.0:1773941523.648467:0:14736:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:3.0:1773941523.648470:0:14736:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:2.0:1773941523.648471:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 10000000:01000000:3.0:1773941523.648472:0:14736:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:3.0:1773941523.648476:0:14736:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000020:00000001:2.0:1773941523.648476:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941523.648480:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941523.648485:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941523.648487:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d035a7800. 02000000:00000001:2.0:1773941523.648491:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941523.648493:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941523.648494:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06505880. 00000100:00000001:2.0:1773941523.648497:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941523.648498:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941523.648500:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1773941523.648502:0:14736:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:3.0:1773941523.648504:0:14736:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941523.648504:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941523.648507:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941523.648508:0:14736:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941523.648512:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:02000000:3.0:1773941523.648516:0:14736:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:2.0:1773941523.648516:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941523.648525:0:14736:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941523.648528:0:14736:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941523.648531:0:14736:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941523.648533:0:14736:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941523.648534:0:14736:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:2.1:1773941523.652008:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d21ec1800. 00010000:00000010:2.1:1773941523.652015:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d00030000. 00000020:00000001:2.0:1773941524.475125:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941524.475133:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941524.475137:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941524.923105:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294915242, at 4294915264 00000100:00000040:0.0:1773941524.923105:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294915241, at 4294915264 00000100:00000010:1.0:1773941524.923125:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d08bbbc60. 00000100:00000010:0.0:1773941524.923132:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d08bbb540. 10000000:00000001:0.0:1773941525.115119:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1773941525.115146:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941525.115153:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941525.115156:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941525.115158:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941525.115159:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1773941525.115163:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:0.0:1773941525.115165:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:0.0:1773941525.115168:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:0.0:1773941525.115171:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1773941525.115173:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:0.0:1773941525.115189:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10837b80. 00000020:00000040:0.0:1773941525.115195:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941525.115202:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941525.115228:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941525.115236:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941525.115246:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065c5800. 00000100:00000001:0.0:1773941525.115255:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941525.115260:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1773941525.115264:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1773941525.115275:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d21ec1800. 00010000:00000001:0.0:1773941525.115282:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1773941525.115303:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d00030600. 00000020:00000001:0.0:1773941525.115307:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:0.0:1773941525.115309:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d00030600 with handle 0x8e86767622e2952 to hash 00000020:00000001:0.0:1773941525.115311:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:0.0:1773941525.115312:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330090849792 : -113743618701824 : ffff988d00030600) 00010000:00000001:0.0:1773941525.115315:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330090849792 : -113743618701824 : ffff988d00030600) 00010000:00000001:0.0:1773941525.115319:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941525.115321:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941525.115325:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e2952 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941525.115334:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e2952 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941525.115345:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e2952 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941525.115350:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1773941525.115352:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941525.115358:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb6600. 00000100:00000001:0.0:1773941525.115361:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330139633152 : -113743569918464 : ffff988d02eb6600) 00000100:00000001:0.0:1773941525.115364:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941525.115366:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367703936 : -113743341847680 : ffff988d10837b80) 00000100:00000001:0.0:1773941525.115372:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941525.115378:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941525.115381:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941525.115389:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941525.115411:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10837b80 x1860112291482368/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941525.115419:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941525.115421:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941525.115425:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941525.115427:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941525.115429:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941525.115432:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10837b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291482368:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941525.115436:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941525.115442:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941525.115445:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941525.115447:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941525.115452:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d02ce6000. 02000000:00000001:0.0:1773941525.115453:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941525.115462:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941525.115466:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0014b840. 00000400:00000010:0.0:1773941525.115473:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833c38. 00000100:00000200:0.0:1773941525.115481:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291482368, portal 25 00000100:00000001:0.0:1773941525.115482:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941525.115483:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367703936 : -113743341847680 : ffff988d10837b80) 00000100:00000040:0.0:1773941525.115487:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10837b80 x1860112291482368/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941541 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941525.115493:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941525.115496:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941525.115498:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833330. 00000100:00000200:0.0:1773941525.115500:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291482368, offset 0 00000400:00000200:0.0:1773941525.115506:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941525.115514:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941525.115522:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941525.115528:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941525.115532:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941525.115536:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941525.115543:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941525.115546:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[46, 45] 00000400:00000200:0.0:1773941525.115568:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:0.0:1773941525.115573:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [157:157:256:4294967295] 192.168.204.124@tcp LPNI seq info [47:47:8:4294967295] 00000400:00000200:0.0:1773941525.115577:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941525.115588:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:0.0:1773941525.115593:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:0.0:1773941525.115599:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a20bd00. 00000800:00000200:0.0:1773941525.115606:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:0.0:1773941525.115615:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:0.0:1773941525.115621:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a20bd00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1773941525.115644:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941525.115650:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941525.115654:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294916711 00000100:00000010:0.0:1773941525.115660:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580b40. 00000100:00000001:0.0:1773941525.115670:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1773941525.115682:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294916711 00000100:00000010:0.0:1773941525.115687:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580420. 00000100:00000001:0.0:1773941525.115692:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1773941525.115697:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941525.115701:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941525.115705:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941525.115709:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1773941525.115714:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6600 going to sleep for 11 seconds 00000100:00000001:0.0:1773941525.115741:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941525.115746:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941525.115877:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941525.115883:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a20bd00. 00000400:00000200:1.0:1773941525.115891:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941525.115898:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941525.115905:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833330 00000400:00000010:1.0:1773941525.115908:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833330. 00000100:00000001:1.0:1773941525.115913:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941525.115919:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10837b80 x1860112291482368/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941541 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941525.115931:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941525.115934:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d10837b80 x1860112291482368/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941541 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941525.115940:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941525.115941:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941525.117339:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941525.117352:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941525.117355:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941525.117356:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941525.117362:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941525.117373:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941525.117378:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941525.117382:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941525.117386:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941525.117389:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324604b00 00000400:00000200:0.0:1773941525.117393:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941525.117397:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xc89 [1] + 224 00000800:00000001:0.0:1773941525.117404:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941525.117422:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941525.117424:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941525.117426:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941525.117431:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941525.117436:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10837b80 x1860112291482368/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941541 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941525.117445:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d10837b80 x1860112291482368/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941541 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941525.117463:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941525.117469:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941525.117482:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941525.117520:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941525.117528:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941525.117530:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941525.117532:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0014b840. 00000400:00000200:2.0:1773941525.117537:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833c38 00000400:00000010:2.0:1773941525.117540:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833c38. 00000100:00000001:2.0:1773941525.117543:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941525.117549:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10837b80 x1860112291482368/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941541 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941525.117592:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10837b80 x1860112291482368/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941541 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941525.117603:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941525.117616:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941525.117620:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941525.117624:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941525.117628:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941525.117632:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941525.117636:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941525.117646:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=26 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941525.117653:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941525.117668:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:2.0:1773941525.117675:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:2.0:1773941525.117679:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941525.117684:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:2.0:1773941525.117688:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1773941525.117694:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1773941525.117701:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1773941525.117707:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941525.117710:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941525.117713:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941525.117738:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:2.0:1773941525.117744:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d10837b80 x1860112291482368/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941541 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941525.117781:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941525.117783:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941525.117785:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941525.117789:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941525.117794:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:2.0:1773941525.117796:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941525.117799:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941525.117801:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941525.117802:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941525.117804:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941525.117806:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941525.117810:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1773941525.117813:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1773941525.117826:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941525.117830:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 246 00000400:00000200:2.0:1773941525.117838:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941525.117844:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:2.0:1773941525.117850:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941525.117854:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941525.117858:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941525.117861:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:2.0:1773941525.117866:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941525.117869:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941525.117872:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941525.117875:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941525.117879:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941525.117883:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:2.0:1773941525.117886:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941525.117889:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941525.117893:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941525.117895:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:2.0:1773941525.117900:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:2.0:1773941525.117905:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:2.0:1773941525.117909:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941525.117913:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941525.117918:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941525.117921:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941525.117925:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941525.117928:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941525.117934:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06505880. 00000020:00000040:2.0:1773941525.117938:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:2.0:1773941525.117943:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941525.117945:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941525.117949:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941525.117951:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941525.117961:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941525.117968:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d035a7800. 00000100:00000001:2.0:1773941525.117973:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941525.118014:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d06505880 x1860112291482496/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941525.118026:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941525.118029:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941525.118036:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e62ba1de] to pc [ptlrpcd_rcv+-1] req@ffff988d06505880 x1860112291482496/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941525.118065:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941525.118070:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:2.0:1773941525.118072:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941525.118076:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941525.118096:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941525.118097:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941525.118100:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6600 going to sleep for 1 seconds 00000100:00000001:2.0:1773941525.118105:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941525.118107:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941525.118114:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10837b80 x1860112291482368/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941541 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941525.118123:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941525.118126:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941525.118127:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941525.118134:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941525.118135:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941525.118138:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941525.118139:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941525.118166:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06505880 x1860112291482496/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941525.118170:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10837b80 x1860112291482368/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941541 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941525.118205:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941525.118209:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941525.118218:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941525.118223:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1773941525.118225:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10837b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291482368:192.168.204.114@tcp:101:ll_cfg_requeue.0 02000000:00000001:3.0:1773941525.118233:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941525.118237:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1773941525.118242:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06505880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291482496:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941525.118243:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941525.118248:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:3.0:1773941525.118252:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941525.118258:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1773941525.118260:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1773941525.118264:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941525.118272:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10837b80 x1860112291482368/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941541 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1773941525.118273:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941525.118282:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03522400. 02000000:00000001:3.0:1773941525.118288:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941525.118294:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941525.118300:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941525.118319:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00000100:00000010:2.0:1773941525.118326:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb6600. 00000400:00000010:3.0:1773941525.118327:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00000100:00000001:2.0:1773941525.118331:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:2.0:1773941525.118335:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000200:3.0:1773941525.118336:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291482496, portal 25 00010000:00000001:2.0:1773941525.118341:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000001:3.0:1773941525.118342:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00000001:2.0:1773941525.118346:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000001:3.0:1773941525.118347:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196580480 : -113743512971136 : ffff988d06505880) 00000020:00000001:2.0:1773941525.118352:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000040:3.0:1773941525.118360:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06505880 x1860112291482496/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941535 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000040:2.0:1773941525.118362:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030600 refcount=4 00000020:00000001:2.0:1773941525.118370:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090849792 : -113743618701824 : ffff988d00030600) 00010000:00000001:2.0:1773941525.118378:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090849792 : -113743618701824 : ffff988d00030600) 00010000:00010000:2.0:1773941525.118385:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e2952 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941525.118406:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00010000:00010000:2.0:1773941525.118418:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e2952 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941525.118419:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941525.118430:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:3.0:1773941525.118437:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac000. 00010000:00000001:2.0:1773941525.118439:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000200:3.0:1773941525.118445:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291482496, offset 0 00010000:00010000:2.0:1773941525.118446:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e2952 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941525.118463:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:3.0:1773941525.118464:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00000001:2.0:1773941525.118468:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:2.0:1773941525.118474:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e2952 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941525.118479:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:2.0:1773941525.118492:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:3.0:1773941525.118493:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:2.0:1773941525.118496:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941525.118502:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:2.0:1773941525.118502:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:3.0:1773941525.118508:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941525.118514:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00010000:2.0:1773941525.118516:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e2952 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941525.118520:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941525.118533:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[46, 47] 00010000:00010000:2.0:1773941525.118542:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d00030600 already unused, calling callback (ffffffffc12b4c20) 00000400:00000200:3.0:1773941525.118547:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941525.118565:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [158:158:256:4294967295] 192.168.204.114@tcp LPNI seq info [48:48:8:4294967295] 10000000:00000001:2.0:1773941525.118567:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:2.0:1773941525.118573:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e2952 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941525.118580:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:2.0:1773941525.118594:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:2.0:1773941525.118599:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:3.0:1773941525.118606:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000020:00000001:2.0:1773941525.118607:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941525.118612:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030600 refcount=5 00000020:00000001:2.0:1773941525.118619:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090849792 : -113743618701824 : ffff988d00030600) 00000800:00000200:3.0:1773941525.118626:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00010000:00000001:2.0:1773941525.118628:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090849792 : -113743618701824 : ffff988d00030600) 00010000:00000001:2.0:1773941525.118637:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000800:00000010:3.0:1773941525.118638:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74500. 00010000:00010000:2.0:1773941525.118643:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e2952 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941525.118656:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 10000000:00000001:2.0:1773941525.118666:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:2.0:1773941525.118671:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e2952 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941525.118674:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:3.0:1773941525.118685:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74500 type 1, nob 616 niov 1 nkiov 1 10000000:01000000:2.0:1773941525.118690:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:2.0:1773941525.118697:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:2.0:1773941525.118699:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941525.118703:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:2.0:1773941525.118706:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:2.0:1773941525.118711:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000100:00000001:3.0:1773941525.118726:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941525.118732:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:3.0:1773941525.118734:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294916714 00010000:00000001:2.0:1773941525.118737:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1773941525.118743:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:3.0:1773941525.118744:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294916711 00000100:00000001:3.0:1773941525.118770:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294916714 00010000:00000001:2.0:1773941525.118773:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941525.118775:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294916711 00000100:00000001:3.0:1773941525.118781:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000040:2.0:1773941525.118781:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d00030600 with handle 0x8e86767622e2952 from hash 00000100:00000001:3.0:1773941525.118795:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941525.118795:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1773941525.118800:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941525.118806:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941525.118813:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1773941525.118814:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941525.118819:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000800:00000200:1.0:1773941525.118819:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1773941525.118820:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941525.118835:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000800:00000010:1.0:1773941525.118836:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74500. 00000100:00000001:3.0:1773941525.118838:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:2.0:1773941525.118838:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000400:00000200:1.0:1773941525.118842:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1773941525.118844:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00010000:00000001:2.0:1773941525.118845:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000100:00000001:3.0:1773941525.118848:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941525.118857:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00010000:00000001:2.0:1773941525.118861:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941525.118862:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941525.118864:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac000 00010000:00000001:2.0:1773941525.118866:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000010:1.0:1773941525.118868:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac000. 00010000:00000001:2.0:1773941525.118874:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941525.118875:0:2393:0:(events.c:34:request_out_callback()) Process entered 10000000:00000001:2.0:1773941525.118882:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1773941525.118883:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06505880 x1860112291482496/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941535 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941525.118904:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00010000:00010000:2.0:1773941525.118908:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e2952 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:1.0:1773941525.118909:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06505880 x1860112291482496/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941535 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1773941525.118949:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941525.118951:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941525.118955:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000100:00000001:1.0:1773941525.118957:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941525.118959:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:2.0:1773941525.118963:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941525.118966:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941525.118968:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1773941525.118972:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e2952 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:2.0:1773941525.118995:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d21ec1800 count: 0 00000100:00000001:1.0:1773941525.119030:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:2.0:1773941525.119062:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941525.119067:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941525.119071:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941525.119079:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10837b80 x1860112291482368/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941541 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941525.119089:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941525.119092:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941525.119096:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d02ce6000. 02000000:00000001:2.0:1773941525.119117:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941525.119120:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941525.119123:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941525.119126:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941525.119129:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065c5800. 02000000:00000001:2.0:1773941525.119135:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941525.119145:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941525.119148:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10837b80. 00000100:00000001:2.0:1773941525.119153:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941525.119154:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:2.0:1773941525.119157:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:2.0:1773941525.119161:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:2.0:1773941525.119165:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:2.0:1773941525.119170:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d05437280. 00000020:00000010:2.0:1773941525.119176:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d0381f200. 00000020:00000010:2.0:1773941525.119183:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d0381e000. 00000040:00000010:2.0:1773941525.119188:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d0381e800. 00000020:00000010:2.0:1773941525.119195:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941525.119203:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d03356200. 80000000:00000010:2.0:1773941525.119209:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d08bbbae0. 00040000:00000010:2.0:1773941525.119216:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d08a4fe00. 20000000:00000010:2.0:1773941525.119222:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d08a4f400. 10000000:00000040:2.0:1773941525.119229:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:2.0:1773941525.119243:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:2.0:1773941525.119250:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d0381e000. 00000040:00000010:2.0:1773941525.119256:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d0381e800. 00000020:00000010:2.0:1773941525.119259:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941525.119265:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d03356200. 80000000:00000010:2.0:1773941525.119269:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d08bbbae0. 00040000:00000010:2.0:1773941525.119274:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d08a4fe00. 20000000:00000010:2.0:1773941525.119280:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d08a4f400. 00000020:00000010:2.0:1773941525.119284:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0381f200. 10000000:00000010:2.0:1773941525.119288:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d05437280. 10000000:01000000:2.0:1773941525.119291:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:2.0:1773941525.119296:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:2.0:1773941525.119316:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:2.0:1773941525.119318:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:2.0:1773941525.119320:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:2.0:1773941525.119322:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941525.119324:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941525.120815:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941525.120826:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941525.120829:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941525.120832:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941525.120838:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941525.120843:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941525.120849:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941525.120852:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941525.120856:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941525.120861:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324604b80 00000400:00000200:0.0:1773941525.120866:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941525.120872:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xc99 [1] + 0 00000800:00000001:0.0:1773941525.120879:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941525.120898:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941525.120902:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941525.120908:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941525.120913:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941525.120920:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06505880 x1860112291482496/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941535 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941525.120934:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d06505880 x1860112291482496/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941535 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941525.120958:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941525.120997:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941525.121003:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941525.121011:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941525.121016:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941525.121022:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941525.121024:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941525.121028:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941525.121032:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941525.121034:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941525.121036:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941525.121044:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06505880 x1860112291482496/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941535 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941525.121057:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06505880 x1860112291482496/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941535 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941525.121065:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941525.121067:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941525.121070:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941525.121073:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941525.121075:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941525.121078:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941525.121091:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941525.121097:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=26 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941525.121101:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941525.121104:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941525.121106:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941525.121109:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941525.121111:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941525.121115:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941525.121120:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06505880 x1860112291482496/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941535 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941525.121143:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941525.121146:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941525.121147:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941525.121150:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941525.121157:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941525.121161:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941525.121163:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941525.121165:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941525.121168:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941525.121171:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941525.121173:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941525.121177:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941525 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941525.121180:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941525.121187:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941525.121190:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941525.121193:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941525.121196:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941525.121197:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941525.121201:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941525.121204:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941525.121207:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941525.121211:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941525.121214:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941525.121217:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941525.121508:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941525.121512:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941525.121519:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06505880 x1860112291482496/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941535 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941525.121530:0:14737:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:2.0:1773941525.121530:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06505880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291482496:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941525.121539:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941525.121541:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1773941525.121547:0:14737:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00000040:2.0:1773941525.121548:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06505880 x1860112291482496/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941535 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941525.121558:0:14737:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941525.121560:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941525.121563:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1773941525.121566:0:14737:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000010:2.0:1773941525.121566:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03522400. 10000000:01000000:3.0:1773941525.121573:0:14737:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000001:2.0:1773941525.121573:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941525.121576:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941525.121578:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941525.121583:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:3.0:1773941525.121584:0:14737:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000010:2.0:1773941525.121585:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d035a7800. 02000000:00000001:2.0:1773941525.121590:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000001:00000001:3.0:1773941525.121591:0:14737:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941525.121592:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941525.121594:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06505880. 00000100:00000001:2.0:1773941525.121596:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941525.121597:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941525.121598:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941525.121606:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000001:00000001:3.0:1773941525.121607:0:14737:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941525.121608:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941525.121621:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000001:00000001:3.0:1773941525.121622:0:14737:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:2.0:1773941525.121623:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941525.121626:0:14737:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941525.121630:0:14737:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941525.121633:0:14737:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941525.121635:0:14737:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941525.121639:0:14737:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:3.0:1773941525.121642:0:14737:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:3.0:1773941525.121644:0:14737:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941525.121646:0:14737:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1773941525.121649:0:14737:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:3.0:1773941525.121651:0:14737:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941525.121653:0:14737:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:3.0:1773941525.121655:0:14737:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:3.0:1773941525.121673:0:14737:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:3.0:1773941525.121674:0:14737:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941525.121676:0:14737:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:3.0:1773941525.121683:0:14737:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:3.0:1773941525.121690:0:14737:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941525.121694:0:14737:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941525.121696:0:14737:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941525.121699:0:14737:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941525.121701:0:14737:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:2.1:1773941525.124033:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d21ec1800. 00010000:00000010:2.1:1773941525.128014:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d00030600. 00000020:00000001:1.0:1773941525.499258:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1773941525.499264:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941525.499285:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1773941525.755109:0:461:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (253) 00000100:00000040:0.0:1773941526.395107:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294916714, at 4294916736 00000100:00000040:1.0:1773941526.395110:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294916714, at 4294916736 00000100:00000010:0.0:1773941526.395132:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580b40. 00000100:00000010:1.0:1773941526.395139:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580420. 00000020:00000001:2.0:1773941526.523147:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941526.523154:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941526.523159:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:2.0:1773941526.779038:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:2.0:1773941526.779046:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941526.779051:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941526.779076:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941526.779078:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941526.779079:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:2.0:1773941526.779082:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:2.0:1773941526.779083:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:2.0:1773941526.779085:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:2.0:1773941526.779088:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:2.0:1773941526.779089:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:2.0:1773941526.779098:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06505880. 00000020:00000040:2.0:1773941526.779103:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941526.779108:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941526.779111:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941526.779116:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941526.779124:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d08a4f400. 00000100:00000001:2.0:1773941526.779132:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941526.779137:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:2.0:1773941526.779140:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:2.0:1773941526.779148:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d071ee100. 00010000:00000001:2.0:1773941526.779154:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:2.0:1773941526.779157:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d089dfa00. 00000020:00000001:2.0:1773941526.779161:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:2.0:1773941526.779163:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d089dfa00 with handle 0x8e86767622e2959 to hash 00000020:00000001:2.0:1773941526.779165:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:2.0:1773941526.779166:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941526.779169:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941526.779173:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941526.779175:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941526.779179:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2959 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941526.779189:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2959 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941526.779204:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2959 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941526.779222:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:2.0:1773941526.779225:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:2.0:1773941526.779233:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d09350540. 00000100:00000001:2.0:1773941526.779236:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330245121344 : -113743464430272 : ffff988d09350540) 00000100:00000001:2.0:1773941526.779240:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941526.779242:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196580480 : -113743512971136 : ffff988d06505880) 00000100:00000001:2.0:1773941526.779248:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941526.779254:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941526.779256:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:2.0:1773941526.779258:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941526.779269:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06505880 x1860112291482624/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941526.779277:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941526.779279:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941526.779283:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941526.779286:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941526.779288:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941526.779290:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06505880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291482624:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941526.779294:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941526.779299:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941526.779302:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941526.779304:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941526.779310:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a7800. 02000000:00000001:2.0:1773941526.779312:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941526.779331:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941526.779337:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000400:00000010:2.0:1773941526.779346:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000200:2.0:1773941526.779356:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291482624, portal 25 00000100:00000001:2.0:1773941526.779359:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941526.779361:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196580480 : -113743512971136 : ffff988d06505880) 00000100:00000040:2.0:1773941526.779367:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06505880 x1860112291482624/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941542 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941526.779388:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941526.779391:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:2.0:1773941526.779393:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516d48. 00000100:00000200:2.0:1773941526.779397:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291482624, offset 0 00000400:00000200:2.0:1773941526.779403:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941526.779412:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941526.779419:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941526.779425:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941526.779429:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941526.779432:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941526.779435:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941526.779437:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[48, 47] 00000400:00000200:2.0:1773941526.779440:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:2.0:1773941526.779443:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [159:159:256:4294967295] 192.168.204.124@tcp LPNI seq info [49:49:8:4294967295] 00000400:00000200:2.0:1773941526.779446:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:2.0:1773941526.779454:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:2.0:1773941526.779460:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:2.0:1773941526.779464:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0db600. 00000800:00000200:2.0:1773941526.779471:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:2.0:1773941526.779478:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:2.0:1773941526.779483:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0db600 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:2.0:1773941526.779499:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941526.779503:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941526.779505:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294918375 00000100:00000010:2.0:1773941526.779509:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d08bbbae0. 00000100:00000001:2.0:1773941526.779515:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1773941526.779522:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294918375 00000100:00000010:2.0:1773941526.779525:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d08bbb300. 00000100:00000001:2.0:1773941526.779527:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1773941526.779534:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941526.779536:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941526.779539:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941526.779541:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:2.0:1773941526.779544:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d09350540 going to sleep for 11 seconds 00000100:00000001:2.0:1773941526.779546:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941526.779550:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941526.779668:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941526.779677:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0db600. 00000400:00000200:1.0:1773941526.779695:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941526.779705:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941526.779713:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516d48 00000400:00000010:1.0:1773941526.779715:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516d48. 00000100:00000001:1.0:1773941526.779719:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941526.779726:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06505880 x1860112291482624/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941542 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941526.779736:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941526.779739:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d06505880 x1860112291482624/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941542 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941526.779745:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941526.779746:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941526.781108:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941526.781121:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941526.781124:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941526.781125:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941526.781131:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941526.781136:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941526.781141:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941526.781145:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941526.781149:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941526.781152:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324604c00 00000400:00000200:0.0:1773941526.781156:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941526.781160:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xca1 [1] + 224 00000800:00000001:0.0:1773941526.781167:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941526.781174:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941526.781177:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941526.781180:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941526.781186:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941526.781193:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06505880 x1860112291482624/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941542 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941526.781220:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d06505880 x1860112291482624/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941542 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941526.781253:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941526.781263:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941526.781267:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941526.781336:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941526.781342:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941526.781344:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941526.781346:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941526.781350:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941526.781351:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941526.781354:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941526.781358:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06505880 x1860112291482624/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941542 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941526.781372:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06505880 x1860112291482624/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941542 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941526.781377:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941526.781379:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941526.781381:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941526.781384:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941526.781386:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941526.781388:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941526.781390:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941526.781396:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=27 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941526.781399:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941526.781402:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:2.0:1773941526.781405:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:2.0:1773941526.781408:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941526.781410:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:2.0:1773941526.781413:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1773941526.781417:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1773941526.781420:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1773941526.781430:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941526.781432:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941526.781434:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941526.781436:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:2.0:1773941526.781439:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d06505880 x1860112291482624/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941542 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941526.781444:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941526.781446:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941526.781447:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941526.781450:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941526.781452:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:2.0:1773941526.781453:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941526.781455:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941526.781456:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941526.781456:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941526.781457:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941526.781459:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941526.781461:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1773941526.781463:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1773941526.781465:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941526.781467:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 248 00000400:00000200:2.0:1773941526.781470:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941526.781474:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:2.0:1773941526.781477:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941526.781479:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941526.781481:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941526.781483:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:2.0:1773941526.781486:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941526.781487:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941526.781489:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941526.781491:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941526.781493:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941526.781495:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:2.0:1773941526.781500:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941526.781502:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941526.781503:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941526.781505:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:2.0:1773941526.781507:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:2.0:1773941526.781510:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:2.0:1773941526.781511:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941526.781513:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941526.781516:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941526.781518:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941526.781519:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941526.781521:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941526.781523:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06506a00. 00000020:00000040:2.0:1773941526.781525:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:2.0:1773941526.781527:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941526.781529:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941526.781531:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941526.781533:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941526.781534:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941526.781538:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d035a4400. 00000100:00000001:2.0:1773941526.781541:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941526.781546:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d06506a00 x1860112291482752/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941526.781551:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941526.781554:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941526.781557:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6b603d8] to pc [ptlrpcd_rcv+-1] req@ffff988d06506a00 x1860112291482752/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941526.781576:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941526.781578:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:2.0:1773941526.781579:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941526.781581:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941526.781586:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941526.781591:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941526.781592:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d09350540 going to sleep for 1 seconds 00000100:00000001:2.0:1773941526.781594:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941526.781596:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941526.781600:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06505880 x1860112291482624/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941542 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941526.781606:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941526.781608:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941526.781610:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941526.781612:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941526.781613:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1773941526.781617:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06505880 x1860112291482624/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941542 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941526.781619:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941526.781622:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941526.781632:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06506a00 x1860112291482752/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941526.781632:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06505880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291482624:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941526.781637:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941526.781642:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941526.781646:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:3.0:1773941526.781650:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941526.781650:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1773941526.781653:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941526.781655:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06505880 x1860112291482624/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941542 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1773941526.781667:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941526.781671:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941526.781672:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941526.781674:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941526.781676:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d09350540. 00000100:00000001:2.0:1773941526.781679:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00100000:3.0:1773941526.781680:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291482752:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941526.781681:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:2.0:1773941526.781698:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000001:3.0:1773941526.781700:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00000001:2.0:1773941526.781702:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:2.0:1773941526.781704:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941526.781706:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089dfa00 refcount=4 00000020:00000001:2.0:1773941526.781707:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941526.781710:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 02000000:00000001:3.0:1773941526.781713:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00010000:2.0:1773941526.781715:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2959 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941526.781716:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941526.781718:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941526.781722:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03522c00. 02000000:00000001:3.0:1773941526.781726:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941526.781734:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:2.0:1773941526.781734:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000010:3.0:1773941526.781739:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00010000:00010000:2.0:1773941526.781741:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2959 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:3.0:1773941526.781745:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00010000:00000001:2.0:1773941526.781748:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000200:3.0:1773941526.781751:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291482752, portal 25 00010000:00010000:2.0:1773941526.781752:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2959 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941526.781758:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00000001:2.0:1773941526.781760:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941526.781762:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584960 : -113743512966656 : ffff988d06506a00) 00010000:00000001:2.0:1773941526.781763:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:2.0:1773941526.781766:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2959 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941526.781773:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000040:3.0:1773941526.781774:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06506a00 x1860112291482752/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941536 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1773941526.781775:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941526.781779:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:2.0:1773941526.781783:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2959 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941526.781795:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d089dfa00 already unused, calling callback (ffffffffc12b4c20) 00000100:00000001:3.0:1773941526.781799:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 10000000:00000001:2.0:1773941526.781799:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:2.0:1773941526.781804:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2959 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:3.0:1773941526.781808:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00000001:2.0:1773941526.781814:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000400:00000010:3.0:1773941526.781815:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac550. 00010000:00000001:2.0:1773941526.781816:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:2.0:1773941526.781819:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000200:3.0:1773941526.781821:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291482752, offset 0 00000020:00000040:2.0:1773941526.781821:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089dfa00 refcount=5 00000020:00000001:2.0:1773941526.781825:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00010000:00000001:2.0:1773941526.781828:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235222528 : -113743474329088 : ffff988d089dfa00) 00000400:00000200:3.0:1773941526.781832:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00000001:2.0:1773941526.781832:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:3.0:1773941526.781838:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:2.0:1773941526.781839:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2959 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941526.781844:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941526.781850:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 10000000:00000001:2.0:1773941526.781850:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:2.0:1773941526.781854:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2959 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941526.781856:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941526.781861:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 10000000:01000000:2.0:1773941526.781865:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:3.0:1773941526.781866:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 10000000:00000040:2.0:1773941526.781870:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000400:00000200:3.0:1773941526.781873:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[48, 49] 10000000:00000001:2.0:1773941526.781874:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941526.781881:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00010000:00010000:2.0:1773941526.781881:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:2.0:1773941526.781883:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:3.0:1773941526.781887:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [160:160:256:4294967295] 192.168.204.114@tcp LPNI seq info [50:50:8:4294967295] 00010000:00000001:2.0:1773941526.781888:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1773941526.781892:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000400:00000200:3.0:1773941526.781895:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:2.0:1773941526.781895:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941526.781899:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941526.781902:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d089dfa00 with handle 0x8e86767622e2959 from hash 00010000:00000001:2.0:1773941526.781906:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000400:00000200:3.0:1773941526.781908:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00010000:00000001:2.0:1773941526.781909:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941526.781912:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941526.781914:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1773941526.781916:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000800:00000200:3.0:1773941526.781920:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00010000:00000001:2.0:1773941526.781920:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:2.0:1773941526.781925:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941526.781926:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000010:3.0:1773941526.781927:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74400. 00010000:00000001:2.0:1773941526.781927:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941526.781932:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941526.781935:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2959 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941526.781938:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00010000:00000001:2.0:1773941526.781941:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941526.781942:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941526.781943:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:2.0:1773941526.781944:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:2.0:1773941526.781947:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941526.781950:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000200:3.0:1773941526.781954:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:3.0:1773941526.781958:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74400 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:2.0:1773941526.781990:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1773941526.781994:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d089dfa00/0x8e86767622e2959 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:2.0:1773941526.782005:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d071ee100 count: 0 00000100:00000001:3.0:1773941526.782011:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941526.782016:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:2.0:1773941526.782018:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:3.0:1773941526.782019:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294918378 00010000:00000001:2.0:1773941526.782022:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941526.782027:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941526.782032:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06505880 x1860112291482624/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941542 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941526.782039:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941526.782042:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1773941526.782044:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294918375 02000000:00000010:2.0:1773941526.782046:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a7800. 02000000:00000001:2.0:1773941526.782051:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1773941526.782052:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294918378 00000020:00000001:2.0:1773941526.782054:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941526.782057:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941526.782058:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294918375 00000020:00000001:2.0:1773941526.782061:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941526.782065:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d08a4f400. 00000100:00000001:3.0:1773941526.782071:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941526.782072:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1773941526.782074:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941526.782075:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1773941526.782079:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941526.782079:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06505880. 00000100:00000001:3.0:1773941526.782085:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941526.782086:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941526.782089:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:2.0:1773941526.782091:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:2.0:1773941526.782093:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000100:00000001:3.0:1773941526.782095:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 10000000:00000001:2.0:1773941526.782095:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000800:00000200:1.0:1773941526.782098:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1773941526.782099:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 10000000:00000010:2.0:1773941526.782102:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d05437280. 00000020:00000010:2.0:1773941526.782115:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d08a4f400. 00000100:00000001:3.0:1773941526.782117:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000010:1.0:1773941526.782117:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74400. 00000100:00000001:3.0:1773941526.782125:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941526.782131:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941526.782133:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1773941526.782134:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1773941526.782134:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d08a4fe00. 00000040:00000010:2.0:1773941526.782150:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d03356200. 00000400:00000200:1.0:1773941526.782157:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000020:00000010:2.0:1773941526.782158:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d09305000. 00000400:00000200:1.0:1773941526.782162:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac550 00000020:00000010:2.0:1773941526.782164:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d0381f200. 00000400:00000010:1.0:1773941526.782166:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac550. 80000000:00000010:2.0:1773941526.782172:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d08bbb000. 00000100:00000001:1.0:1773941526.782174:0:2393:0:(events.c:34:request_out_callback()) Process entered 00040000:00000010:2.0:1773941526.782178:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d0381e800. 20000000:00000010:2.0:1773941526.782183:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d0381e000. 00000100:00000200:1.0:1773941526.782185:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06506a00 x1860112291482752/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941536 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000040:2.0:1773941526.782196:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:2.0:1773941526.782201:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941526.782206:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000020:00000010:2.0:1773941526.782207:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d08a4fe00. 00000040:00000010:2.0:1773941526.782214:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d03356200. 00000100:00000040:1.0:1773941526.782216:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06506a00 x1860112291482752/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941536 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1773941526.782219:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941526.782226:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d0381f200. 80000000:00000010:2.0:1773941526.782232:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d08bbb000. 00040000:00000010:2.0:1773941526.782238:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d0381e800. 00000100:00000001:1.0:1773941526.782238:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941526.782246:0:2393:0:(events.c:68:request_out_callback()) Process leaving 20000000:00000010:2.0:1773941526.782301:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d0381e000. 00000020:00000010:2.0:1773941526.782305:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d08a4f400. 10000000:00000010:2.0:1773941526.782308:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d05437280. 10000000:01000000:2.0:1773941526.782310:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:2.0:1773941526.782312:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:2.0:1773941526.782314:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:2.0:1773941526.782316:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:2.0:1773941526.782317:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:2.0:1773941526.782318:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941526.782320:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941526.783876:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941526.783884:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941526.783886:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941526.783887:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941526.783891:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941526.783896:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941526.783900:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941526.783903:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941526.783905:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941526.783908:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324604c80 00000400:00000200:0.0:1773941526.783910:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941526.783913:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xca9 [1] + 0 00000800:00000001:0.0:1773941526.783917:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941526.783922:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941526.783924:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941526.783926:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941526.783929:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941526.783934:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06506a00 x1860112291482752/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941536 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941526.783955:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d06506a00 x1860112291482752/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941536 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941526.783984:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941526.783989:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941526.783992:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941526.784010:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941526.784013:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941526.784016:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941526.784017:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941526.784018:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941526.784023:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941526.784024:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941526.784026:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941526.784031:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06506a00 x1860112291482752/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941536 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941526.784039:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06506a00 x1860112291482752/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941536 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941526.784062:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941526.784064:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941526.784066:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941526.784074:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941526.784076:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941526.784078:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941526.784080:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941526.784083:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=27 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941526.784086:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941526.784088:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941526.784090:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941526.784092:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941526.784094:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941526.784096:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941526.784100:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06506a00 x1860112291482752/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941536 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941526.784105:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941526.784107:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941526.784109:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941526.784111:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941526.784115:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941526.784117:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941526.784118:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941526.784120:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941526.784122:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941526.784124:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941526.784126:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941526.784129:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941526 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941526.784131:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941526.784138:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941526.784139:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941526.784141:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941526.784143:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941526.784144:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941526.784147:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941526.784150:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941526.784152:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941526.784160:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941526.784163:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941526.784165:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941526.784404:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941526.784408:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941526.784415:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06506a00 x1860112291482752/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941536 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941526.784416:0:14738:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00080000:3.0:1773941526.784442:0:14738:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:2.0:1773941526.784446:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291482752:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00100000:3.0:1773941526.784448:0:14738:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:3.0:1773941526.784456:0:14738:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:2.0:1773941526.784457:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 10000000:01000000:3.0:1773941526.784459:0:14738:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000001:2.0:1773941526.784462:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000001:00000001:3.0:1773941526.784464:0:14738:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000040:2.0:1773941526.784467:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06506a00 x1860112291482752/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941536 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000001:00000001:3.0:1773941526.784469:0:14738:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941526.784475:0:14738:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941526.784477:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:3.0:1773941526.784478:0:14738:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:2.0:1773941526.784480:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:3.0:1773941526.784481:0:14738:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000010:2.0:1773941526.784484:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03522c00. 00000001:00000001:3.0:1773941526.784485:0:14738:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941526.784489:0:14738:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941526.784490:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1773941526.784492:0:14738:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941526.784496:0:14738:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000020:00000001:2.0:1773941526.784501:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:3.0:1773941526.784502:0:14738:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000020:00000040:2.0:1773941526.784506:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941526.784507:0:14738:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941526.784511:0:14738:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941526.784512:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00080000:3.0:1773941526.784515:0:14738:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 02000000:00000010:2.0:1773941526.784516:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d035a4400. 00000100:00000001:3.0:1773941526.784521:0:14738:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:2.0:1773941526.784523:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 10000000:01000000:3.0:1773941526.784524:0:14738:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 02000000:00000001:2.0:1773941526.784525:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 10000000:00000040:3.0:1773941526.784528:0:14738:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00000010:2.0:1773941526.784529:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06506a00. 00000100:00000001:2.0:1773941526.784535:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941526.784537:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941526.784539:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941526.784542:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941526.784544:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941526.784546:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941526.784547:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1773941526.784549:0:14738:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:3.0:1773941526.784568:0:14738:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941526.784571:0:14738:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:3.0:1773941526.784574:0:14738:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:3.0:1773941526.784577:0:14738:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941526.784580:0:14738:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941526.784581:0:14738:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941526.784582:0:14738:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941526.784584:0:14738:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:2.1:1773941526.787995:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d071ee100. 00010000:00000010:2.1:1773941526.788002:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d089dfa00. 00000020:00000001:1.0:1773941527.547322:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1773941527.547360:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941527.547366:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1773941528.059084:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294918378, at 4294918400 00000100:00000040:1.0:1773941528.059089:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294918378, at 4294918400 00000100:00000010:1.0:1773941528.059115:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d08bbb300. 00000100:00000010:0.0:1773941528.059124:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d08bbbae0. 10000000:00000001:0.0:1773941528.379115:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1773941528.379125:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941528.379130:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941528.379133:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941528.379134:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941528.379136:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1773941528.379139:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:0.0:1773941528.379141:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:0.0:1773941528.379144:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:0.0:1773941528.379146:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1773941528.379148:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:0.0:1773941528.379161:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10835180. 00000020:00000040:0.0:1773941528.379167:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941528.379173:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941528.379177:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941528.379183:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941528.379193:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065c4e00. 00000100:00000001:0.0:1773941528.379201:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941528.379206:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1773941528.379209:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1773941528.379220:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d21ec1400. 00010000:00000001:0.0:1773941528.379226:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1773941528.379230:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d00030a00. 00000020:00000001:0.0:1773941528.379234:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:0.0:1773941528.379237:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d00030a00 with handle 0x8e86767622e2960 to hash 00000020:00000001:0.0:1773941528.379239:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:0.0:1773941528.379241:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330090850816 : -113743618700800 : ffff988d00030a00) 00010000:00000001:0.0:1773941528.379244:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330090850816 : -113743618700800 : ffff988d00030a00) 00010000:00000001:0.0:1773941528.379248:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941528.379251:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941528.379255:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00030a00/0x8e86767622e2960 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941528.379266:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d00030a00/0x8e86767622e2960 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941528.379306:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d00030a00/0x8e86767622e2960 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941528.379315:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1773941528.379319:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941528.379328:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb6600. 00000100:00000001:0.0:1773941528.379331:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330139633152 : -113743569918464 : ffff988d02eb6600) 00000100:00000001:0.0:1773941528.379335:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941528.379337:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367693184 : -113743341858432 : ffff988d10835180) 00000100:00000001:0.0:1773941528.379344:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941528.379351:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941528.379353:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941528.379355:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941528.379368:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10835180 x1860112291482880/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941528.379376:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941528.379379:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941528.379383:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941528.379386:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941528.379389:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941528.379392:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10835180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291482880:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941528.379396:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941528.379402:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941528.379404:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941528.379406:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941528.379411:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d02ce7400. 02000000:00000001:0.0:1773941528.379413:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941528.379422:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941528.379426:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0014b370. 00000400:00000010:0.0:1773941528.379433:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833cc0. 00000100:00000200:0.0:1773941528.379441:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291482880, portal 25 00000100:00000001:0.0:1773941528.379443:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941528.379452:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367693184 : -113743341858432 : ffff988d10835180) 00000100:00000040:0.0:1773941528.379457:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10835180 x1860112291482880/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941544 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941528.379464:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941528.379467:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941528.379469:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833770. 00000100:00000200:0.0:1773941528.379472:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291482880, offset 0 00000400:00000200:0.0:1773941528.379478:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941528.379487:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941528.379496:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941528.379502:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941528.379507:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941528.379511:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941528.379513:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941528.379516:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[50, 49] 00000400:00000200:0.0:1773941528.379536:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:0.0:1773941528.379540:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [161:161:256:4294967295] 192.168.204.124@tcp LPNI seq info [51:51:8:4294967295] 00000400:00000200:0.0:1773941528.379544:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941528.379556:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:0.0:1773941528.379563:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:0.0:1773941528.379568:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a20bd00. 00000800:00000200:0.0:1773941528.379576:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:0.0:1773941528.379586:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:0.0:1773941528.379591:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a20bd00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1773941528.379613:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941528.379618:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941528.379621:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294919975 00000100:00000010:0.0:1773941528.379625:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580b40. 00000100:00000001:0.0:1773941528.379634:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1773941528.379642:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294919975 00000100:00000010:0.0:1773941528.379651:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580a20. 00000100:00000001:0.0:1773941528.379698:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1773941528.379702:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941528.379705:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941528.379708:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941528.379710:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1773941528.379714:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6600 going to sleep for 11 seconds 00000100:00000001:0.0:1773941528.379717:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941528.379720:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941528.379844:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941528.379851:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a20bd00. 00000400:00000200:1.0:1773941528.379857:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941528.379864:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941528.379870:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833770 00000400:00000010:1.0:1773941528.379872:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833770. 00000100:00000001:1.0:1773941528.379877:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941528.379882:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10835180 x1860112291482880/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941544 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941528.379893:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941528.379896:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d10835180 x1860112291482880/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941544 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941528.379903:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941528.379905:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941528.381087:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941528.381100:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941528.381104:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941528.381107:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941528.381114:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941528.381120:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941528.381127:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941528.381132:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941528.381138:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941528.381143:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324604d00 00000400:00000200:0.0:1773941528.381148:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941528.381154:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xcb1 [1] + 224 00000800:00000001:0.0:1773941528.381163:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941528.381173:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941528.381176:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941528.381180:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941528.381185:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941528.381190:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10835180 x1860112291482880/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941544 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941528.381202:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d10835180 x1860112291482880/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941544 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941528.381225:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941528.381233:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941528.381242:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941528.381295:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941528.381302:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941528.381303:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941528.381306:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0014b370. 00000400:00000200:2.0:1773941528.381310:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833cc0 00000400:00000010:2.0:1773941528.381311:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833cc0. 00000100:00000001:2.0:1773941528.381313:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941528.381318:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10835180 x1860112291482880/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941544 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941528.381326:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10835180 x1860112291482880/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941544 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941528.381332:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941528.381333:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941528.381335:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941528.381337:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941528.381339:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941528.381341:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941528.381343:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941528.381348:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=29 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941528.381350:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941528.381353:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:2.0:1773941528.381356:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:2.0:1773941528.381359:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941528.381369:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:2.0:1773941528.381371:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1773941528.381375:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1773941528.381379:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1773941528.381382:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941528.381384:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941528.381391:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941528.381392:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:2.0:1773941528.381395:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d10835180 x1860112291482880/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941544 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941528.381401:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941528.381403:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941528.381405:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941528.381407:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941528.381409:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:2.0:1773941528.381410:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941528.381412:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941528.381413:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941528.381414:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941528.381415:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941528.381416:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941528.381418:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1773941528.381420:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1773941528.381422:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941528.381424:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 249 00000400:00000200:2.0:1773941528.381428:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941528.381431:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:2.0:1773941528.381435:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941528.381437:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941528.381439:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941528.381441:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:2.0:1773941528.381444:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941528.381449:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941528.381452:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941528.381454:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941528.381456:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941528.381458:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:2.0:1773941528.381460:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941528.381462:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941528.381463:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941528.381464:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:2.0:1773941528.381467:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:2.0:1773941528.381470:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:2.0:1773941528.381472:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941528.381474:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941528.381477:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941528.381479:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941528.381481:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941528.381482:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941528.381486:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06506a00. 00000020:00000040:2.0:1773941528.381489:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:2.0:1773941528.381493:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941528.381495:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941528.381498:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941528.381500:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941528.381502:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941528.381507:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d035a7800. 00000100:00000001:2.0:1773941528.381512:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941528.381518:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d06506a00 x1860112291483008/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941528.381528:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941528.381531:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941528.381536:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6b603d8] to pc [ptlrpcd_rcv+-1] req@ffff988d06506a00 x1860112291483008/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941528.381575:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941528.381577:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:2.0:1773941528.381579:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941528.381581:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941528.381583:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941528.381584:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941528.381585:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6600 going to sleep for 1 seconds 00000100:00000001:2.0:1773941528.381587:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941528.381589:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941528.381594:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10835180 x1860112291482880/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941544 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941528.381599:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941528.381601:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941528.381603:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941528.381605:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941528.381608:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10835180 x1860112291482880/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941544 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941528.381613:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10835180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291482880:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941528.381616:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941528.381618:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941528.381619:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:2.0:1773941528.381620:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941528.381622:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10835180 x1860112291482880/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941544 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941528.381626:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941528.381628:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb6600. 00000100:00000001:2.0:1773941528.381631:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:2.0:1773941528.381632:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:2.0:1773941528.381634:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:2.0:1773941528.381641:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:2.0:1773941528.381643:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941528.381645:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030a00 refcount=4 00000020:00000001:2.0:1773941528.381647:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090850816 : -113743618700800 : ffff988d00030a00) 00010000:00000001:2.0:1773941528.381649:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090850816 : -113743618700800 : ffff988d00030a00) 00010000:00010000:2.0:1773941528.381653:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d00030a00/0x8e86767622e2960 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941528.381674:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00010000:00010000:2.0:1773941528.381678:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d00030a00/0x8e86767622e2960 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941528.381683:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000001:3.0:1773941528.381686:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00010000:2.0:1773941528.381687:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00030a00/0x8e86767622e2960 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941528.381695:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00010000:00000001:2.0:1773941528.381697:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941528.381699:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:2.0:1773941528.381701:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d00030a00/0x8e86767622e2960 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941528.381705:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941528.381707:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941528.381709:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:2.0:1773941528.381712:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d00030a00/0x8e86767622e2960 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941528.381715:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00010000:00010000:2.0:1773941528.381718:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d00030a00 already unused, calling callback (ffffffffc12b4c20) 10000000:00000001:2.0:1773941528.381738:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:2.0:1773941528.381741:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d00030a00/0x8e86767622e2960 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:3.0:1773941528.381745:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06506a00 x1860112291483008/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1773941528.381747:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:2.0:1773941528.381758:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:2.0:1773941528.381762:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941528.381766:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030a00 refcount=5 00000020:00000001:2.0:1773941528.381769:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090850816 : -113743618700800 : ffff988d00030a00) 00010000:00000001:2.0:1773941528.381775:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090850816 : -113743618700800 : ffff988d00030a00) 00000100:00000001:3.0:1773941528.381776:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:2.0:1773941528.381779:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000100:00000001:3.0:1773941528.381781:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941528.381783:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d00030a00/0x8e86767622e2960 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941528.381793:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 10000000:00000001:2.0:1773941528.381793:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:2.0:1773941528.381797:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d00030a00/0x8e86767622e2960 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941528.381803:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 10000000:01000000:2.0:1773941528.381804:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:2.0:1773941528.381814:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 02000000:00000001:3.0:1773941528.381815:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941528.381816:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941528.381824:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:2.0:1773941528.381827:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000100:00100000:3.0:1773941528.381830:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291483008:192.168.204.114@tcp:250:ll_cfg_requeue.0 00010000:00000001:2.0:1773941528.381830:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1773941528.381833:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1773941528.381836:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941528.381838:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941528.381843:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d00030a00 with handle 0x8e86767622e2960 from hash 00000100:00000001:3.0:1773941528.381845:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00000001:2.0:1773941528.381847:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 02000000:00000001:3.0:1773941528.381863:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00000001:2.0:1773941528.381866:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 02000000:00000001:3.0:1773941528.381867:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941528.381871:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 02000000:00000001:3.0:1773941528.381873:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:2.0:1773941528.381874:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1773941528.381877:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:2.0:1773941528.381879:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 02000000:00000010:3.0:1773941528.381881:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03522400. 00010000:00000001:2.0:1773941528.381883:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941528.381891:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 02000000:00000001:3.0:1773941528.381892:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941528.381909:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941528.381920:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941528.381930:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941528.381936:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00010000:00010000:2.0:1773941528.381950:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d00030a00/0x8e86767622e2960 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:3.0:1773941528.381953:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00000100:00000200:3.0:1773941528.381963:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291483008, portal 25 00000100:00000001:3.0:1773941528.381966:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941528.381968:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584960 : -113743512966656 : ffff988d06506a00) 00010000:00000001:2.0:1773941528.381986:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941528.381990:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941528.381992:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:2.0:1773941528.381995:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:2.0:1773941528.381998:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941528.382015:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941528.382017:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000040:3.0:1773941528.382027:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06506a00 x1860112291483008/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941538 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941528.382036:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941528.382039:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:3.0:1773941528.382042:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac4c8. 00000100:00000200:3.0:1773941528.382046:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291483008, offset 0 00000400:00000200:3.0:1773941528.382053:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00010000:2.0:1773941528.382062:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d00030a00/0x8e86767622e2960 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941528.382063:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000040:2.0:1773941528.382078:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d21ec1400 count: 0 00000400:00000200:3.0:1773941528.382081:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941528.382090:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:2.0:1773941528.382092:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941528.382095:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941528.382097:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000400:00000200:3.0:1773941528.382102:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000100:00000040:2.0:1773941528.382108:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10835180 x1860112291482880/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941544 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941528.382117:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000100:00000001:2.0:1773941528.382119:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941528.382122:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941528.382125:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d02ce7400. 02000000:00000001:2.0:1773941528.382127:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941528.382130:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941528.382133:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941528.382138:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 00000400:00000200:3.0:1773941528.382139:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 02000000:00000010:2.0:1773941528.382142:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065c4e00. 02000000:00000001:2.0:1773941528.382159:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941528.382160:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941528.382162:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10835180. 00000400:00000200:3.0:1773941528.382169:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[50, 51] 00000100:00000001:2.0:1773941528.382169:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941528.382174:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:2.0:1773941528.382179:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:2.0:1773941528.382185:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:2.0:1773941528.382188:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000400:00000200:3.0:1773941528.382190:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 10000000:00000010:2.0:1773941528.382193:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d05437280. 00000020:00000010:2.0:1773941528.382199:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d0381e000. 00000400:00000200:3.0:1773941528.382202:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [162:162:256:4294967295] 192.168.204.114@tcp LPNI seq info [52:52:8:4294967295] 00000020:00000010:2.0:1773941528.382212:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d0381e800. 00000400:00000200:3.0:1773941528.382221:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000040:00000010:2.0:1773941528.382222:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d0381f200. 00000020:00000010:2.0:1773941528.382228:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941528.382234:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d03356200. 00000400:00000200:3.0:1773941528.382236:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 80000000:00000010:2.0:1773941528.382242:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d08bbb000. 00000800:00000200:3.0:1773941528.382250:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00040000:00000010:2.0:1773941528.382254:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d08a4f400. 00000800:00000010:3.0:1773941528.382257:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74400. 20000000:00000010:2.0:1773941528.382261:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d08a4fe00. 00000800:00000200:3.0:1773941528.382266:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 10000000:00000040:2.0:1773941528.382268:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:2.0:1773941528.382273:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000800:00000200:3.0:1773941528.382280:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000020:00000010:2.0:1773941528.382282:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d0381e800. 00000800:00000200:3.0:1773941528.382288:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74400 type 1, nob 616 niov 1 nkiov 1 00000040:00000010:2.0:1773941528.382290:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d0381f200. 00000020:00000010:2.0:1773941528.382296:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941528.382299:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d03356200. 80000000:00000010:2.0:1773941528.382302:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d08bbb000. 00040000:00000010:2.0:1773941528.382307:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d08a4f400. 00000100:00000001:3.0:1773941528.382325:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 20000000:00000010:2.0:1773941528.382328:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d08a4fe00. 00000400:00000200:3.0:1773941528.382333:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000020:00000010:2.0:1773941528.382334:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0381e000. 00000100:00000001:3.0:1773941528.382339:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294919978 10000000:00000010:2.0:1773941528.382340:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d05437280. 10000000:01000000:2.0:1773941528.382343:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:2.0:1773941528.382353:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:2.0:1773941528.382355:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:2.0:1773941528.382357:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 00000100:00000001:3.0:1773941528.382358:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294919975 10000000:00000001:2.0:1773941528.382359:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:2.0:1773941528.382368:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 00000100:00000001:3.0:1773941528.382372:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294919978 10000000:00000001:2.0:1773941528.382373:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000100:00000001:3.0:1773941528.382389:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294919975 00000100:00000001:3.0:1773941528.382397:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941528.382399:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941528.382402:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941528.382404:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941528.382411:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941528.382412:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1773941528.382414:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941528.382415:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941528.382417:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941528.382418:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941528.382499:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941528.382504:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74400. 00000400:00000200:1.0:1773941528.382508:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941528.382514:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941528.382534:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac4c8 00000400:00000010:1.0:1773941528.382536:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac4c8. 00000100:00000001:1.0:1773941528.382540:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941528.382545:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06506a00 x1860112291483008/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941538 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941528.382554:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941528.382557:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06506a00 x1860112291483008/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941538 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941528.382564:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941528.382566:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941528.384047:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941528.384058:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941528.384061:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941528.384064:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941528.384068:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941528.384071:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941528.384077:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941528.384079:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941528.384083:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941528.384087:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324604d80 00000400:00000200:0.0:1773941528.384090:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941528.384094:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xcc1 [1] + 0 00000800:00000001:0.0:1773941528.384109:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941528.384115:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941528.384117:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941528.384119:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941528.384122:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941528.384126:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06506a00 x1860112291483008/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941538 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941528.384134:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d06506a00 x1860112291483008/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941538 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941528.384150:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941528.384154:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941528.384156:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941528.384190:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941528.384194:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941528.384199:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941528.384201:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941528.384205:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941528.384218:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941528.384220:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941528.384224:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941528.384231:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06506a00 x1860112291483008/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941538 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941528.384242:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06506a00 x1860112291483008/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941538 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941528.384250:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941528.384251:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941528.384254:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941528.384256:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941528.384259:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941528.384260:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941528.384263:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941528.384269:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=29 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941528.384272:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941528.384275:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941528.384276:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941528.384279:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941528.384281:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941528.384283:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941528.384287:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06506a00 x1860112291483008/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941538 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941528.384293:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941528.384296:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941528.384298:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941528.384300:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941528.384306:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941528.384309:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941528.384311:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941528.384313:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941528.384316:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941528.384319:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941528.384321:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941528.384346:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941528 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941528.384349:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941528.384354:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941528.384356:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941528.384359:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941528.384360:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941528.384362:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941528.384364:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941528.384367:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941528.384369:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941528.384372:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941528.384374:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941528.384376:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941528.384728:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941528.384733:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941528.384740:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06506a00 x1860112291483008/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941538 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941528.384749:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291483008:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941528.384752:0:14740:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941528.384755:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1773941528.384761:0:14740:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00000001:2.0:1773941528.384771:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00100000:3.0:1773941528.384772:0:14740:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:3.0:1773941528.384777:0:14740:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941528.384779:0:14740:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000040:2.0:1773941528.384779:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06506a00 x1860112291483008/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941538 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000001:00000001:3.0:1773941528.384789:0:14740:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:2.0:1773941528.384797:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:3.0:1773941528.384798:0:14740:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941528.384800:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:3.0:1773941528.384802:0:14740:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941528.384804:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03522400. 00000001:00000001:3.0:1773941528.384806:0:14740:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941528.384814:0:14740:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941528.384815:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941528.384819:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:3.0:1773941528.384820:0:14740:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941528.384821:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941528.384830:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 10000000:00000001:3.0:1773941528.384831:0:14740:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941528.384833:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d035a7800. 00000100:00000001:3.0:1773941528.384849:0:14740:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000001:2.0:1773941528.384853:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941528.384856:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941528.384858:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06506a00. 00000100:00080000:3.0:1773941528.384859:0:14740:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941528.384861:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941528.384865:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941528.384871:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941528.384872:0:14740:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941528.384876:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941528.384877:0:14740:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:2.0:1773941528.384879:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941528.384883:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941528.384884:0:14740:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941528.384886:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1773941528.384896:0:14740:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:3.0:1773941528.384902:0:14740:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941528.384904:0:14740:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:3.0:1773941528.384907:0:14740:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:3.0:1773941528.384949:0:14740:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:3.0:1773941528.384951:0:14740:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941528.384953:0:14740:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:3.0:1773941528.384957:0:14740:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:3.0:1773941528.384997:0:14740:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941528.385000:0:14740:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941528.385003:0:14740:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941528.385006:0:14740:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941528.385008:0:14740:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:2.1:1773941528.388030:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d21ec1400. 00010000:00000010:2.1:1773941528.388036:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d00030a00. 00000020:00000001:2.0:1773941528.571287:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941528.571301:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941528.571311:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1773941529.595132:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1773941529.595140:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941529.595145:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941529.659154:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294919978, at 4294920000 00000100:00000040:0.0:1773941529.659154:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294919978, at 4294920000 00000100:00000010:0.0:1773941529.659174:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580b40. 00000100:00000010:1.0:1773941529.659175:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580a20. 10000000:00000001:2.0:1773941530.235040:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:2.0:1773941530.235050:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941530.235055:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941530.235058:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941530.235059:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941530.235061:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:2.0:1773941530.235064:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:2.0:1773941530.235066:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:2.0:1773941530.235068:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:2.0:1773941530.235071:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:2.0:1773941530.235072:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:2.0:1773941530.235085:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06506a00. 00000020:00000040:2.0:1773941530.235091:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941530.235115:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941530.235118:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941530.235131:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941530.235141:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d08a4fe00. 00000100:00000001:2.0:1773941530.235149:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941530.235154:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:2.0:1773941530.235157:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:2.0:1773941530.235166:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d071ee100. 00010000:00000001:2.0:1773941530.235171:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:2.0:1773941530.235174:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d089de800. 00000020:00000001:2.0:1773941530.235179:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:2.0:1773941530.235181:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d089de800 with handle 0x8e86767622e2967 to hash 00000020:00000001:2.0:1773941530.235183:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:2.0:1773941530.235184:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330235217920 : -113743474333696 : ffff988d089de800) 00010000:00000001:2.0:1773941530.235187:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330235217920 : -113743474333696 : ffff988d089de800) 00010000:00000001:2.0:1773941530.235191:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941530.235194:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941530.235197:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2967 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941530.235206:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2967 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941530.235217:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2967 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941530.235222:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:2.0:1773941530.235224:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:2.0:1773941530.235231:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d09350540. 00000100:00000001:2.0:1773941530.235232:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330245121344 : -113743464430272 : ffff988d09350540) 00000100:00000001:2.0:1773941530.235235:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941530.235236:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584960 : -113743512966656 : ffff988d06506a00) 00000100:00000001:2.0:1773941530.235246:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941530.235261:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941530.235267:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:2.0:1773941530.235269:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941530.235281:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06506a00 x1860112291483136/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941530.235289:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941530.235291:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941530.235295:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941530.235298:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941530.235299:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941530.235302:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291483136:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941530.235305:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941530.235312:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941530.235314:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941530.235317:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941530.235322:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a7800. 02000000:00000001:2.0:1773941530.235323:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941530.235332:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941530.235337:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000400:00000010:2.0:1773941530.235344:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000200:2.0:1773941530.235352:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291483136, portal 25 00000100:00000001:2.0:1773941530.235354:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941530.235355:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584960 : -113743512966656 : ffff988d06506a00) 00000100:00000040:2.0:1773941530.235359:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06506a00 x1860112291483136/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941546 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941530.235365:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941530.235368:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:2.0:1773941530.235370:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516ee0. 00000100:00000200:2.0:1773941530.235374:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291483136, offset 0 00000400:00000200:2.0:1773941530.235380:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941530.235389:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941530.235397:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941530.235403:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941530.235407:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941530.235414:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941530.235417:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941530.235420:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[52, 51] 00000400:00000200:2.0:1773941530.235424:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:2.0:1773941530.235429:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [163:163:256:4294967295] 192.168.204.124@tcp LPNI seq info [53:53:8:4294967295] 00000400:00000200:2.0:1773941530.235445:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:2.0:1773941530.235459:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:2.0:1773941530.235467:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:2.0:1773941530.235475:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0db600. 00000800:00000200:2.0:1773941530.235485:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:2.0:1773941530.235497:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:2.0:1773941530.235504:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0db600 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:2.0:1773941530.235524:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941530.235527:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941530.235542:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294921831 00000100:00000010:2.0:1773941530.235564:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d08bbb000. 00000100:00000001:2.0:1773941530.235572:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1773941530.235582:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294921831 00000100:00000010:2.0:1773941530.235586:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d08bbbea0. 00000100:00000001:2.0:1773941530.235589:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1773941530.235595:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941530.235597:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941530.235599:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941530.235602:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:2.0:1773941530.235604:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d09350540 going to sleep for 11 seconds 00000100:00000001:2.0:1773941530.235607:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941530.235611:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941530.235742:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941530.235748:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0db600. 00000400:00000200:1.0:1773941530.235755:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941530.235760:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941530.235766:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516ee0 00000400:00000010:1.0:1773941530.235768:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516ee0. 00000100:00000001:1.0:1773941530.235772:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941530.235776:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06506a00 x1860112291483136/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941546 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941530.235786:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941530.235789:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d06506a00 x1860112291483136/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941546 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941530.235795:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941530.235797:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941530.237216:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941530.237228:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941530.237230:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941530.237232:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941530.237237:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941530.237242:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941530.237256:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941530.237259:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941530.237271:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941530.237278:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324604e00 00000400:00000200:0.0:1773941530.237281:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941530.237285:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xcc9 [1] + 224 00000800:00000001:0.0:1773941530.237292:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941530.237300:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941530.237302:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941530.237304:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941530.237308:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941530.237312:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06506a00 x1860112291483136/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941546 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941530.237321:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d06506a00 x1860112291483136/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941546 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941530.237335:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941530.237343:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941530.237345:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941530.237372:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941530.237379:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941530.237381:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941530.237383:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941530.237387:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941530.237397:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941530.237399:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941530.237404:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06506a00 x1860112291483136/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941546 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941530.237412:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06506a00 x1860112291483136/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941546 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941530.237418:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941530.237420:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941530.237423:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941530.237426:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941530.237429:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941530.237432:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941530.237436:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941530.237443:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=31 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941530.237446:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941530.237450:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:2.0:1773941530.237454:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:2.0:1773941530.237458:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941530.237462:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:2.0:1773941530.237477:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1773941530.237482:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1773941530.237488:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1773941530.237492:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941530.237495:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941530.237498:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941530.237500:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:2.0:1773941530.237504:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d06506a00 x1860112291483136/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941546 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941530.237514:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941530.237516:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941530.237518:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941530.237526:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941530.237529:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:2.0:1773941530.237530:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941530.237531:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941530.237548:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941530.237550:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941530.237551:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941530.237552:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941530.237555:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1773941530.237557:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1773941530.237559:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941530.237562:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 251 00000400:00000200:2.0:1773941530.237566:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941530.237571:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:2.0:1773941530.237574:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941530.237577:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941530.237579:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941530.237581:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:2.0:1773941530.237585:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941530.237587:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941530.237589:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941530.237591:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941530.237593:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941530.237595:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:2.0:1773941530.237597:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941530.237598:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941530.237599:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941530.237601:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:2.0:1773941530.237603:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:2.0:1773941530.237606:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:2.0:1773941530.237608:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941530.237610:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941530.237613:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941530.237650:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941530.237653:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941530.237666:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941530.237669:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06505880. 00000020:00000040:2.0:1773941530.237672:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:2.0:1773941530.237675:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941530.237676:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941530.237678:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941530.237679:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941530.237681:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941530.237685:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d035a6400. 00000100:00000001:2.0:1773941530.237688:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941530.237693:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d06505880 x1860112291483264/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941530.237699:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941530.237701:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941530.237705:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e62ba1de] to pc [ptlrpcd_rcv+-1] req@ffff988d06505880 x1860112291483264/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941530.237724:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941530.237726:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:2.0:1773941530.237728:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941530.237730:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941530.237732:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941530.237733:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941530.237734:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d09350540 going to sleep for 1 seconds 00000100:00000001:2.0:1773941530.237736:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941530.237738:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941530.237742:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06506a00 x1860112291483136/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941546 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941530.237748:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941530.237750:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941530.237751:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941530.237753:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941530.237761:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06506a00 x1860112291483136/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941546 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941530.237768:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291483136:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941530.237773:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941530.237773:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941530.237776:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941530.237779:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:3.0:1773941530.237780:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941530.237781:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1773941530.237785:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941530.237786:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06506a00 x1860112291483136/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941546 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941530.237793:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941530.237797:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d09350540. 00000100:00000040:3.0:1773941530.237798:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06505880 x1860112291483264/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941530.237800:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:2.0:1773941530.237803:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:2.0:1773941530.237808:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:2.0:1773941530.237812:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000001:3.0:1773941530.237815:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:2.0:1773941530.237816:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:3.0:1773941530.237820:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941530.237820:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089de800 refcount=4 00000020:00000001:2.0:1773941530.237822:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235217920 : -113743474333696 : ffff988d089de800) 02000000:00000001:3.0:1773941530.237827:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00010000:00000001:2.0:1773941530.237827:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235217920 : -113743474333696 : ffff988d089de800) 02000000:00000001:3.0:1773941530.237832:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00010000:00010000:2.0:1773941530.237833:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2967 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941530.237839:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941530.237847:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00100000:3.0:1773941530.237848:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06505880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291483264:192.168.204.114@tcp:250:ll_cfg_requeue.0 00010000:00010000:2.0:1773941530.237852:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2967 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941530.237856:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941530.237861:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00000001:2.0:1773941530.237862:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 02000000:00000001:3.0:1773941530.237866:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941530.237867:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2967 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941530.237873:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941530.237888:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03520000. 00010000:00000001:2.0:1773941530.237893:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 02000000:00000001:3.0:1773941530.237894:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941530.237897:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:2.0:1773941530.237900:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2967 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941530.237901:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:2.0:1773941530.237909:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000010:3.0:1773941530.237910:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00010000:00000001:2.0:1773941530.237911:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941530.237917:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000010:3.0:1773941530.237920:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00010000:00010000:2.0:1773941530.237920:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2967 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:3.0:1773941530.237928:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291483264, portal 25 00010000:00010000:2.0:1773941530.237929:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d089de800 already unused, calling callback (ffffffffc12b4c20) 00000100:00000001:3.0:1773941530.237934:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 10000000:00000001:2.0:1773941530.237935:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000001:3.0:1773941530.237939:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196580480 : -113743512971136 : ffff988d06505880) 10000000:00010000:2.0:1773941530.237940:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2967 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941530.237949:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:2.0:1773941530.237952:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000040:3.0:1773941530.237953:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06505880 x1860112291483264/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941540 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1773941530.237954:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941530.237978:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089de800 refcount=5 00000020:00000001:2.0:1773941530.238003:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235217920 : -113743474333696 : ffff988d089de800) 00010000:00000001:2.0:1773941530.238006:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235217920 : -113743474333696 : ffff988d089de800) 00010000:00000001:2.0:1773941530.238009:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:2.0:1773941530.238012:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2967 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:2.0:1773941530.238017:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:2.0:1773941530.238021:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2967 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1773941530.238038:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:2.0:1773941530.238041:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:2.0:1773941530.238050:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941530.238057:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:2.0:1773941530.238061:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000100:00000001:3.0:1773941530.238063:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:2.0:1773941530.238066:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000100:00000040:3.0:1773941530.238071:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00000001:2.0:1773941530.238071:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1773941530.238075:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941530.238077:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1773941530.238078:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac440. 00000020:00000040:2.0:1773941530.238080:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d089de800 with handle 0x8e86767622e2967 from hash 00010000:00000001:2.0:1773941530.238085:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000100:00000200:3.0:1773941530.238087:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291483264, offset 0 00010000:00000001:2.0:1773941530.238087:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941530.238090:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941530.238092:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1773941530.238094:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:2.0:1773941530.238095:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000400:00000200:3.0:1773941530.238101:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00000001:2.0:1773941530.238101:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941530.238104:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941530.238106:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941530.238108:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941530.238112:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:2.0:1773941530.238114:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2967 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941530.238121:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:2.0:1773941530.238122:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941530.238125:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941530.238128:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000400:00000200:3.0:1773941530.238129:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:2.0:1773941530.238130:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:2.0:1773941530.238135:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941530.238137:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:3.0:1773941530.238138:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00000001:2.0:1773941530.238139:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1773941530.238143:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2967 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941530.238145:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941530.238154:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000040:2.0:1773941530.238156:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d071ee100 count: 0 00000400:00000200:3.0:1773941530.238162:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[52, 53] 00000400:00000200:3.0:1773941530.238168:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00010000:00000001:2.0:1773941530.238171:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:3.0:1773941530.238174:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [164:164:256:4294967295] 192.168.204.114@tcp LPNI seq info [54:54:8:4294967295] 00010000:00000001:2.0:1773941530.238175:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941530.238180:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941530.238188:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06506a00 x1860112291483136/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941546 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941530.238189:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941530.238205:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000100:00000001:2.0:1773941530.238214:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000800:00000200:3.0:1773941530.238220:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 02000000:00000001:2.0:1773941530.238220:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941530.238226:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a7800. 00000800:00000010:3.0:1773941530.238229:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74500. 02000000:00000001:2.0:1773941530.238230:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941530.238235:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941530.238237:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000800:00000200:3.0:1773941530.238243:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000020:00000001:2.0:1773941530.238243:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941530.238248:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d08a4fe00. 02000000:00000001:2.0:1773941530.238254:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000800:00000200:3.0:1773941530.238256:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 02000000:00000001:2.0:1773941530.238269:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000800:00000200:3.0:1773941530.238289:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74500 type 1, nob 616 niov 1 nkiov 1 00000100:00000010:2.0:1773941530.238290:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06506a00. 00000100:00000001:2.0:1773941530.238295:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941530.238298:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:2.0:1773941530.238300:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:2.0:1773941530.238301:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:2.0:1773941530.238303:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000100:00000001:3.0:1773941530.238307:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 10000000:00000010:2.0:1773941530.238307:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d05437280. 00000020:00000010:2.0:1773941530.238313:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d08a4fe00. 00000400:00000200:3.0:1773941530.238315:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:3.0:1773941530.238320:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294921834 00000020:00000010:2.0:1773941530.238321:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d08a4f400. 00000100:00000001:3.0:1773941530.238328:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294921831 00000040:00000010:2.0:1773941530.238330:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d03356200. 00000100:00000001:3.0:1773941530.238334:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294921834 00000020:00000010:2.0:1773941530.238339:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d09305000. 00000100:00000001:3.0:1773941530.238340:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294921831 00000100:00000001:3.0:1773941530.238357:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941530.238360:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1773941530.238362:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d0381e000. 00000100:00000001:3.0:1773941530.238370:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 80000000:00000010:2.0:1773941530.238371:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d08bbb120. 00000100:00000001:3.0:1773941530.238374:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00040000:00000010:2.0:1773941530.238379:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d0381f200. 00000100:00000001:3.0:1773941530.238383:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 20000000:00000010:2.0:1773941530.238385:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d0381e800. 00000100:00000001:3.0:1773941530.238386:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1773941530.238392:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000200:1.0:1773941530.238393:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 10000000:00000040:2.0:1773941530.238394:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 00000100:00000001:3.0:1773941530.238395:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000800:00000010:1.0:1773941530.238399:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74500. 10000000:00000001:2.0:1773941530.238400:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941530.238401:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941530.238406:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1773941530.238408:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941530.238412:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000020:00000010:2.0:1773941530.238422:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d08a4f400. 00000400:00000200:1.0:1773941530.238425:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac440 00000400:00000010:1.0:1773941530.238429:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac440. 00000040:00000010:2.0:1773941530.238431:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d03356200. 00000100:00000001:1.0:1773941530.238432:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000020:00000010:2.0:1773941530.238441:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d09305000. 00000100:00000200:1.0:1773941530.238441:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06505880 x1860112291483264/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941540 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1773941530.238450:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d0381e000. 00000100:00000001:1.0:1773941530.238452:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941530.238457:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06505880 x1860112291483264/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941540 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 80000000:00000010:2.0:1773941530.238458:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d08bbb120. 00040000:00000010:2.0:1773941530.238466:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d0381f200. 00000100:00000001:1.0:1773941530.238467:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941530.238476:0:2393:0:(events.c:68:request_out_callback()) Process leaving 20000000:00000010:2.0:1773941530.238484:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d0381e800. 00000020:00000010:2.0:1773941530.238489:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d08a4fe00. 10000000:00000010:2.0:1773941530.238494:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d05437280. 10000000:01000000:2.0:1773941530.238498:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:2.0:1773941530.238501:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:2.0:1773941530.238503:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:2.0:1773941530.238505:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:2.0:1773941530.238506:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:2.0:1773941530.238507:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941530.238510:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941530.240220:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941530.240227:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941530.240229:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941530.240232:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941530.240235:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941530.240239:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941530.240243:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941530.240246:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941530.240248:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941530.240250:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324604e80 00000400:00000200:0.0:1773941530.240253:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941530.240256:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xcd1 [1] + 0 00000800:00000001:0.0:1773941530.240260:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941530.240270:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941530.240272:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941530.240275:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941530.240278:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941530.240281:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06505880 x1860112291483264/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941540 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941530.240289:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d06505880 x1860112291483264/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941540 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941530.240303:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941530.240308:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941530.240310:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941530.240345:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941530.240348:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941530.240353:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941530.240355:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941530.240357:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941530.240363:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941530.240365:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941530.240368:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941530.240375:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06505880 x1860112291483264/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941540 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941530.240394:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06505880 x1860112291483264/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941540 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941530.240402:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941530.240404:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941530.240406:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941530.240408:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941530.240410:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941530.240411:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941530.240413:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941530.240417:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=31 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941530.240420:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941530.240423:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941530.240425:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941530.240427:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941530.240430:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941530.240433:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941530.240437:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06505880 x1860112291483264/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941540 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941530.240458:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941530.240460:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941530.240462:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941530.240465:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941530.240470:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941530.240472:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941530.240474:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941530.240476:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941530.240478:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941530.240481:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941530.240495:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941530.240498:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941530 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941530.240502:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941530.240509:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941530.240512:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941530.240513:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941530.240516:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941530.240517:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941530.240522:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941530.240524:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941530.240527:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941530.240531:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941530.240535:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941530.240537:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941530.240789:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941530.240794:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941530.240801:0:14741:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000040:2.0:1773941530.240801:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06505880 x1860112291483264/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941540 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1773941530.240810:0:14741:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:3.0:1773941530.240815:0:14741:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00100000:2.0:1773941530.240816:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06505880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291483264:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941530.240828:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941530.240830:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941530.240832:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06505880 x1860112291483264/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941540 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941530.240838:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941530.240840:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1773941530.240842:0:14741:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000010:2.0:1773941530.240843:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03520000. 10000000:01000000:3.0:1773941530.240847:0:14741:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000001:2.0:1773941530.240848:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941530.240851:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941530.240853:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000001:00000001:3.0:1773941530.240854:0:14741:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:2.0:1773941530.240856:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941530.240858:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d035a6400. 00000001:00000001:3.0:1773941530.240860:0:14741:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941530.240861:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941530.240862:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941530.240863:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06505880. 00000100:00000001:2.0:1773941530.240865:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941530.240866:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941530.240867:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:3.0:1773941530.240873:0:14741:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941530.240879:0:14741:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:2.0:1773941530.240880:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000001:00000001:3.0:1773941530.240881:0:14741:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941530.240883:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941530.240885:0:14741:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941530.240887:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941530.240890:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941530.240891:0:14741:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941530.240895:0:14741:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941530.240900:0:14741:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:3.0:1773941530.240903:0:14741:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:3.0:1773941530.240906:0:14741:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941530.240907:0:14741:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1773941530.240909:0:14741:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:3.0:1773941530.240911:0:14741:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941530.240912:0:14741:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:3.0:1773941530.240914:0:14741:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:3.0:1773941530.240926:0:14741:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:3.0:1773941530.240927:0:14741:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941530.240928:0:14741:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:3.0:1773941530.240931:0:14741:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:3.0:1773941530.240934:0:14741:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941530.240937:0:14741:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941530.240938:0:14741:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941530.240940:0:14741:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941530.240942:0:14741:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:2.1:1773941530.243987:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d071ee100. 00010000:00000010:2.1:1773941530.243995:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d089de800. 00000020:00000001:2.0:1773941530.619253:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941530.619261:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941530.619267:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941530.875120:0:461:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (258) 10000000:00000001:2.0:1773941531.323143:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:2.0:1773941531.323157:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941531.323167:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941531.323172:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941531.323175:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941531.323178:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:2.0:1773941531.323183:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:2.0:1773941531.323186:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:2.0:1773941531.323189:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:2.0:1773941531.323193:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:2.0:1773941531.323196:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:2.0:1773941531.323213:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06505880. 00000020:00000040:2.0:1773941531.323218:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941531.323225:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941531.323228:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941531.323235:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941531.323245:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d0381e800. 00000100:00000001:2.0:1773941531.323253:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941531.323258:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:2.0:1773941531.323261:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:2.0:1773941531.323285:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d071ee100. 00010000:00000001:2.0:1773941531.323293:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:2.0:1773941531.323296:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d089de800. 00000020:00000001:2.0:1773941531.323301:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:2.0:1773941531.323304:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d089de800 with handle 0x8e86767622e296e to hash 00000020:00000001:2.0:1773941531.323306:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:2.0:1773941531.323308:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330235217920 : -113743474333696 : ffff988d089de800) 00010000:00000001:2.0:1773941531.323311:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330235217920 : -113743474333696 : ffff988d089de800) 00010000:00000001:2.0:1773941531.323315:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941531.323336:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941531.323341:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e296e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941531.323352:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e296e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941531.323376:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e296e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941531.323382:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:2.0:1773941531.323384:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:2.0:1773941531.323391:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d09350540. 00000100:00000001:2.0:1773941531.323393:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330245121344 : -113743464430272 : ffff988d09350540) 00000100:00000001:2.0:1773941531.323396:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941531.323397:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196580480 : -113743512971136 : ffff988d06505880) 00000100:00000001:2.0:1773941531.323402:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941531.323407:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941531.323410:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:2.0:1773941531.323412:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941531.323424:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06505880 x1860112291483392/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941531.323433:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941531.323446:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941531.323450:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941531.323454:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941531.323458:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941531.323462:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06505880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291483392:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941531.323468:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941531.323477:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941531.323481:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941531.323485:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941531.323491:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a6400. 02000000:00000001:2.0:1773941531.323501:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941531.323513:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941531.323519:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000400:00000010:2.0:1773941531.323531:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000200:2.0:1773941531.323539:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291483392, portal 25 00000100:00000001:2.0:1773941531.323541:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941531.323543:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196580480 : -113743512971136 : ffff988d06505880) 00000100:00000040:2.0:1773941531.323549:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06505880 x1860112291483392/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941547 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941531.323557:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941531.323560:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:2.0:1773941531.323562:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516c38. 00000100:00000200:2.0:1773941531.323566:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291483392, offset 0 00000400:00000200:2.0:1773941531.323573:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941531.323582:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941531.323591:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941531.323597:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941531.323621:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941531.323625:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941531.323628:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941531.323641:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[54, 53] 00000400:00000200:2.0:1773941531.323645:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:2.0:1773941531.323649:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [165:165:256:4294967295] 192.168.204.124@tcp LPNI seq info [55:55:8:4294967295] 00000400:00000200:2.0:1773941531.323653:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:2.0:1773941531.323664:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:2.0:1773941531.323670:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:2.0:1773941531.323676:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0db900. 00000800:00000200:2.0:1773941531.323684:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:2.0:1773941531.323692:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:2.0:1773941531.323707:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0db900 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:2.0:1773941531.323724:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941531.323734:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941531.323737:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294922919 00000100:00000001:2.0:1773941531.323747:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294921834 00000100:00000001:2.0:1773941531.323752:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294922919 00000100:00000001:2.0:1773941531.323756:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294921834 00000100:00000001:2.0:1773941531.323760:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941531.323762:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941531.323765:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941531.323767:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:2.0:1773941531.323770:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d09350540 going to sleep for 11 seconds 00000100:00000001:2.0:1773941531.323772:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941531.323775:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941531.323864:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941531.323872:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0db900. 00000400:00000200:1.0:1773941531.323881:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941531.323890:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941531.323899:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516c38 00000400:00000010:1.0:1773941531.323902:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516c38. 00000100:00000001:1.0:1773941531.323908:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941531.323915:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06505880 x1860112291483392/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941547 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941531.323930:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941531.323936:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d06505880 x1860112291483392/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941547 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941531.323944:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941531.323946:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941531.325678:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941531.325692:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941531.325695:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941531.325697:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941531.325702:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941531.325708:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941531.325714:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941531.325718:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941531.325723:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941531.325727:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324604f00 00000400:00000200:0.0:1773941531.325731:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941531.325735:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xcd9 [1] + 224 00000800:00000001:0.0:1773941531.325743:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941531.325752:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941531.325755:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941531.325758:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941531.325762:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941531.325769:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06505880 x1860112291483392/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941547 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941531.325801:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d06505880 x1860112291483392/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941547 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941531.325840:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941531.325850:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941531.325855:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941531.325877:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941531.325884:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941531.325886:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941531.325889:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941531.325892:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941531.325894:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941531.325896:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941531.325901:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06505880 x1860112291483392/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941547 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941531.325911:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06505880 x1860112291483392/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941547 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941531.325917:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941531.325918:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941531.325921:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941531.325935:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941531.325937:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941531.325939:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941531.325941:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941531.325948:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=32 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941531.325951:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941531.325953:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:2.0:1773941531.325957:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:2.0:1773941531.325981:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941531.325984:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:2.0:1773941531.325987:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1773941531.326003:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1773941531.326007:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1773941531.326010:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941531.326013:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941531.326015:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941531.326017:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:2.0:1773941531.326020:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d06505880 x1860112291483392/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941547 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941531.326026:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941531.326027:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941531.326029:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941531.326031:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941531.326034:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:2.0:1773941531.326035:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941531.326036:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941531.326037:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941531.326038:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941531.326040:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941531.326041:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941531.326043:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1773941531.326045:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1773941531.326048:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941531.326050:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 253 00000400:00000200:2.0:1773941531.326054:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941531.326058:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:2.0:1773941531.326061:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941531.326064:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941531.326066:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941531.326067:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:2.0:1773941531.326075:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941531.326093:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941531.326095:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941531.326098:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941531.326100:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941531.326102:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:2.0:1773941531.326105:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941531.326107:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941531.326108:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941531.326110:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:2.0:1773941531.326113:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:2.0:1773941531.326116:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:2.0:1773941531.326118:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941531.326131:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941531.326133:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941531.326136:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941531.326138:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941531.326139:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941531.326143:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06506a00. 00000020:00000040:2.0:1773941531.326145:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:2.0:1773941531.326148:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941531.326149:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941531.326151:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941531.326153:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941531.326155:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941531.326159:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d035a7800. 00000100:00000001:2.0:1773941531.326162:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941531.326167:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d06506a00 x1860112291483520/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941531.326174:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941531.326177:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941531.326181:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6b603d8] to pc [ptlrpcd_rcv+-1] req@ffff988d06506a00 x1860112291483520/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941531.326214:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941531.326217:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:2.0:1773941531.326230:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941531.326233:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941531.326235:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941531.326236:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941531.326238:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d09350540 going to sleep for 1 seconds 00000100:00000001:2.0:1773941531.326240:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941531.326242:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941531.326248:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06505880 x1860112291483392/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941547 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941531.326259:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941531.326264:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941531.326269:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941531.326272:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941531.326279:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1773941531.326279:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06505880 x1860112291483392/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941547 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941531.326288:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941531.326294:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00100000:2.0:1773941531.326317:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06505880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291483392:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000040:3.0:1773941531.326319:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06506a00 x1860112291483520/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941531.326332:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941531.326338:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941531.326341:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941531.326342:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941531.326350:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1773941531.326352:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 02000000:00000001:3.0:1773941531.326355:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941531.326357:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1773941531.326359:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941531.326375:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291483520:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941531.326383:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000040:2.0:1773941531.326388:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06505880 x1860112291483392/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941547 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941531.326398:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941531.326406:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000010:2.0:1773941531.326406:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d09350540. 02000000:00000001:3.0:1773941531.326412:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941531.326417:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1773941531.326418:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:2.0:1773941531.326423:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000010:3.0:1773941531.326424:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03523800. 02000000:00000001:3.0:1773941531.326430:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941531.326435:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000400:00000200:3.0:1773941531.326440:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941531.326448:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00010000:00000001:2.0:1773941531.326455:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000010:3.0:1773941531.326457:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00000020:00000001:2.0:1773941531.326462:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000200:3.0:1773941531.326466:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291483520, portal 25 00000020:00000040:2.0:1773941531.326468:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089de800 refcount=4 00000100:00000001:3.0:1773941531.326474:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941531.326478:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584960 : -113743512966656 : ffff988d06506a00) 00000020:00000001:2.0:1773941531.326478:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235217920 : -113743474333696 : ffff988d089de800) 00010000:00000001:2.0:1773941531.326495:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235217920 : -113743474333696 : ffff988d089de800) 00000100:00000040:3.0:1773941531.326497:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06506a00 x1860112291483520/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941541 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:2.0:1773941531.326512:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e296e lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941531.326514:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941531.326525:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:3.0:1773941531.326532:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac880. 00010000:00000001:2.0:1773941531.326537:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000200:3.0:1773941531.326539:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291483520, offset 0 00010000:00010000:2.0:1773941531.326549:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e296e lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941531.326559:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00000001:2.0:1773941531.326570:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000400:00000200:3.0:1773941531.326575:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:2.0:1773941531.326575:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e296e lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941531.326591:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:3.0:1773941531.326593:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:2.0:1773941531.326596:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:3.0:1773941531.326617:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00010000:2.0:1773941531.326619:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e296e lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941531.326624:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941531.326637:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:2.0:1773941531.326639:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941531.326642:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941531.326643:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:2.0:1773941531.326653:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:3.0:1773941531.326658:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[54, 55] 00010000:00010000:2.0:1773941531.326659:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e296e lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941531.326690:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941531.326703:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [166:166:256:4294967295] 192.168.204.114@tcp LPNI seq info [56:56:8:4294967295] 00010000:00010000:2.0:1773941531.326709:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d089de800 already unused, calling callback (ffffffffc12b4c20) 00000400:00000200:3.0:1773941531.326717:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 10000000:00000001:2.0:1773941531.326718:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:2.0:1773941531.326730:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e296e lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941531.326753:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00010000:00000001:2.0:1773941531.326760:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000800:00000200:3.0:1773941531.326761:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:3.0:1773941531.326766:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74f00. 00010000:00000001:2.0:1773941531.326772:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000800:00000200:3.0:1773941531.326788:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000020:00000001:2.0:1773941531.326792:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941531.326794:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089de800 refcount=5 00000020:00000001:2.0:1773941531.326801:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235217920 : -113743474333696 : ffff988d089de800) 00010000:00000001:2.0:1773941531.326805:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235217920 : -113743474333696 : ffff988d089de800) 00010000:00000001:2.0:1773941531.326809:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000800:00000200:3.0:1773941531.326811:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00010000:00010000:2.0:1773941531.326813:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e296e lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941531.326818:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74f00 type 1, nob 616 niov 1 nkiov 1 10000000:00000001:2.0:1773941531.326837:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:2.0:1773941531.326840:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e296e lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1773941531.326844:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:2.0:1773941531.326848:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000100:00000001:3.0:1773941531.326853:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941531.326853:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941531.326856:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:2.0:1773941531.326859:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:3.0:1773941531.326861:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:2.0:1773941531.326864:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000100:00000001:3.0:1773941531.326866:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294922922 00010000:00000001:2.0:1773941531.326868:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1773941531.326872:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941531.326877:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941531.326883:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294922919 00000020:00000040:2.0:1773941531.326884:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d089de800 with handle 0x8e86767622e296e from hash 00010000:00000001:2.0:1773941531.326892:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1773941531.326895:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941531.326900:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294922922 00010000:00000001:2.0:1773941531.326900:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941531.326906:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1773941531.326910:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000100:00000001:3.0:1773941531.326913:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294922919 00010000:00000001:2.0:1773941531.326913:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:2.0:1773941531.326921:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941531.326924:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:2.0:1773941531.326924:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941531.326929:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941531.326931:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941531.326935:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941531.326938:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941531.326939:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e296e lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941531.326943:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:1.0:1773941531.326951:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941531.326959:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74f00. 00010000:00000001:2.0:1773941531.326962:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941531.326963:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941531.326965:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:2.0:1773941531.326968:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:2.0:1773941531.326971:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941531.326973:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941531.326974:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:1.0:1773941531.326996:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00010000:2.0:1773941531.327016:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e296e lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:1.0:1773941531.327016:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941531.327044:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac880 00010000:00000040:2.0:1773941531.327045:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d071ee100 count: 0 00000400:00000010:1.0:1773941531.327052:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac880. 00000100:00000001:1.0:1773941531.327060:0:2393:0:(events.c:34:request_out_callback()) Process entered 00010000:00000001:2.0:1773941531.327061:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941531.327064:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941531.327068:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000200:1.0:1773941531.327069:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06506a00 x1860112291483520/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941541 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941531.327075:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06505880 x1860112291483392/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941547 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941531.327115:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:1.0:1773941531.327116:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1773941531.327119:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941531.327123:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a6400. 00000100:00000040:1.0:1773941531.327127:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06506a00 x1860112291483520/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941541 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941531.327128:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:2.0:1773941531.327129:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941531.327139:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:3.0:1773941531.327140:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000020:00000040:2.0:1773941531.327148:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941531.327156:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941531.327157:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941531.327158:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941531.327160:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d0381e800. 00000100:00000001:3.0:1773941531.327161:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941531.327162:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1773941531.327166:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941531.327169:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1773941531.327176:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941531.327176:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941531.327181:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06505880. 00000100:00000001:2.0:1773941531.327189:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941531.327192:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:2.0:1773941531.327195:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:2.0:1773941531.327197:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:2.0:1773941531.327200:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:2.0:1773941531.327204:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d05437280. 00000020:00000010:2.0:1773941531.327209:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d0381e800. 00000020:00000010:2.0:1773941531.327220:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d0381f200. 00000040:00000010:2.0:1773941531.327223:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d0381e000. 00000020:00000010:2.0:1773941531.327228:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941531.327234:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d03356200. 80000000:00000010:2.0:1773941531.327238:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d08bbb120. 00040000:00000010:2.0:1773941531.327242:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d08a4fe00. 20000000:00000010:2.0:1773941531.327247:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d08a4f400. 10000000:00000040:2.0:1773941531.327251:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:2.0:1773941531.327253:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:2.0:1773941531.327258:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d0381f200. 00000040:00000010:2.0:1773941531.327264:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d0381e000. 00000020:00000010:2.0:1773941531.327268:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941531.327273:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d03356200. 80000000:00000010:2.0:1773941531.327278:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d08bbb120. 00040000:00000010:2.0:1773941531.327282:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d08a4fe00. 20000000:00000010:2.0:1773941531.327288:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d08a4f400. 00000020:00000010:2.0:1773941531.327291:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0381e800. 10000000:00000010:2.0:1773941531.327295:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d05437280. 10000000:01000000:2.0:1773941531.327299:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:2.0:1773941531.327303:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:2.0:1773941531.327305:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:2.0:1773941531.327308:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:2.0:1773941531.327310:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:2.0:1773941531.327312:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941531.327315:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941531.329060:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941531.329068:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941531.329071:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941531.329073:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941531.329077:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941531.329082:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941531.329087:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941531.329090:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941531.329094:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941531.329097:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324604f80 00000400:00000200:0.0:1773941531.329101:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941531.329105:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xce1 [1] + 0 00000800:00000001:0.0:1773941531.329109:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941531.329115:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941531.329117:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941531.329120:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941531.329124:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941531.329129:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06506a00 x1860112291483520/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941541 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941531.329151:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d06506a00 x1860112291483520/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941541 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941531.329166:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941531.329171:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941531.329173:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941531.329204:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941531.329209:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941531.329216:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941531.329218:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941531.329222:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941531.329228:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941531.329231:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941531.329234:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941531.329242:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06506a00 x1860112291483520/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941541 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941531.329253:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06506a00 x1860112291483520/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941541 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941531.329259:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941531.329261:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941531.329263:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941531.329285:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941531.329287:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941531.329289:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941531.329291:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941531.329295:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=32 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941531.329299:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941531.329301:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941531.329303:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941531.329305:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941531.329307:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941531.329309:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941531.329313:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06506a00 x1860112291483520/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941541 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941531.329331:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941531.329333:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941531.329335:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941531.329337:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941531.329343:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941531.329345:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941531.329346:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941531.329348:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941531.329350:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941531.329351:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941531.329353:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941531.329357:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941531 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941531.329359:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941531.329366:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941531.329368:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941531.329369:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941531.329371:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941531.329373:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941531.329376:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941531.329379:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941531.329381:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941531.329389:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941531.329392:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941531.329394:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941531.329674:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941531.329680:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941531.329689:0:14742:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000040:2.0:1773941531.329689:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06506a00 x1860112291483520/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941541 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1773941531.329701:0:14742:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:2.0:1773941531.329707:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291483520:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00100000:3.0:1773941531.329708:0:14742:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941531.329722:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941531.329730:0:14742:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941531.329733:0:14742:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000001:2.0:1773941531.329733:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000001:00000001:3.0:1773941531.329739:0:14742:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941531.329742:0:14742:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941531.329745:0:14742:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941531.329747:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06506a00 x1860112291483520/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941541 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000001:00000001:3.0:1773941531.329748:0:14742:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941531.329751:0:14742:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941531.329753:0:14742:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941531.329758:0:14742:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941531.329762:0:14742:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941531.329768:0:14742:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941531.329769:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1773941531.329771:0:14742:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 02000000:00000001:2.0:1773941531.329773:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1773941531.329774:0:14742:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941531.329779:0:14742:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941531.329779:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03523800. 00000100:00080000:3.0:1773941531.329782:0:14742:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:3.0:1773941531.329785:0:14742:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941531.329787:0:14742:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 02000000:00000001:2.0:1773941531.329787:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 10000000:00000040:3.0:1773941531.329790:0:14742:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000020:00000001:2.0:1773941531.329792:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941531.329796:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941531.329799:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941531.329802:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d035a7800. 00000100:00080000:3.0:1773941531.329806:0:14742:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 02000000:00000001:2.0:1773941531.329807:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 10000000:00000001:3.0:1773941531.329809:0:14742:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941531.329811:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1773941531.329812:0:14742:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000010:2.0:1773941531.329825:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06506a00. 00000100:02000000:3.0:1773941531.329826:0:14742:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:3.0:1773941531.329832:0:14742:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941531.329834:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000020:00000001:3.0:1773941531.329835:0:14742:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941531.329840:0:14742:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941531.329841:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1773941531.329844:0:14742:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941531.329847:0:14742:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941531.329847:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941531.329856:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941531.329859:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941531.329863:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941531.329866:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:2.1:1773941531.333009:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d071ee100. 00010000:00000010:2.1:1773941531.333018:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d089de800. 00000020:00000001:1.0:1773941531.643143:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1773941531.643150:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941531.643156:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1773941532.603045:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294922922, at 4294922944 00000100:00000010:0.0:1773941532.603063:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d08bbb000. 00000100:00000040:1.0:1773941532.603068:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294922922, at 4294922944 00000100:00000010:1.0:1773941532.603079:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d08bbbea0. 00000020:00000001:2.0:1773941532.667187:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941532.667194:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941532.667198:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:0.0:1773941533.179085:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1773941533.179118:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941533.179126:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941533.179129:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941533.179131:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941533.179133:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1773941533.179137:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:0.0:1773941533.179139:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:0.0:1773941533.179142:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:0.0:1773941533.179145:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1773941533.179147:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:0.0:1773941533.179157:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10836a00. 00000020:00000040:0.0:1773941533.179164:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941533.179169:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941533.179174:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941533.179180:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941533.179189:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065c4a00. 00000100:00000001:0.0:1773941533.179199:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941533.179206:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1773941533.179209:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1773941533.179219:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d21ec1800. 00010000:00000001:0.0:1773941533.179226:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1773941533.179230:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d00031e00. 00000020:00000001:0.0:1773941533.179236:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:0.0:1773941533.179239:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d00031e00 with handle 0x8e86767622e2975 to hash 00000020:00000001:0.0:1773941533.179242:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:0.0:1773941533.179244:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330090855936 : -113743618695680 : ffff988d00031e00) 00010000:00000001:0.0:1773941533.179248:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330090855936 : -113743618695680 : ffff988d00031e00) 00010000:00000001:0.0:1773941533.179252:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941533.179256:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941533.179262:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2975 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941533.179275:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2975 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941533.179302:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2975 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941533.179310:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1773941533.179313:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941533.179321:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb6840. 00000100:00000001:0.0:1773941533.179325:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330139633728 : -113743569917888 : ffff988d02eb6840) 00000100:00000001:0.0:1773941533.179328:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941533.179330:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367699456 : -113743341852160 : ffff988d10836a00) 00000100:00000001:0.0:1773941533.179336:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941533.179341:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941533.179344:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941533.179347:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941533.179360:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10836a00 x1860112291483648/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941533.179370:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941533.179372:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941533.179377:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941533.179381:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941533.179384:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941533.179388:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10836a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291483648:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941533.179394:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941533.179400:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941533.179403:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941533.179406:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941533.179412:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d02ce7000. 02000000:00000001:0.0:1773941533.179415:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941533.179424:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941533.179429:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0014b790. 00000400:00000010:0.0:1773941533.179438:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833a18. 00000100:00000200:0.0:1773941533.179447:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291483648, portal 25 00000100:00000001:0.0:1773941533.179449:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941533.179456:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367699456 : -113743341852160 : ffff988d10836a00) 00000100:00000040:0.0:1773941533.179463:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10836a00 x1860112291483648/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941549 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941533.179472:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941533.179475:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941533.179479:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d108334c8. 00000100:00000200:0.0:1773941533.179483:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291483648, offset 0 00000400:00000200:0.0:1773941533.179491:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941533.179501:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941533.179511:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941533.179518:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941533.179523:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941533.179527:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941533.179530:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941533.179534:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[56, 55] 00000400:00000200:0.0:1773941533.179538:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:0.0:1773941533.179543:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [167:167:256:4294967295] 192.168.204.124@tcp LPNI seq info [57:57:8:4294967295] 00000400:00000200:0.0:1773941533.179549:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941533.179562:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:0.0:1773941533.179585:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:0.0:1773941533.179593:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a20bd00. 00000800:00000200:0.0:1773941533.179602:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:0.0:1773941533.179612:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:0.0:1773941533.179618:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a20bd00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1773941533.179642:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941533.179647:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941533.179650:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294924775 00000100:00000010:0.0:1773941533.179655:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580b40. 00000100:00000001:0.0:1773941533.179664:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1773941533.179673:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294924775 00000100:00000010:0.0:1773941533.179687:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580d80. 00000100:00000001:0.0:1773941533.179691:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1773941533.179696:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941533.179700:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941533.179703:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941533.179705:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1773941533.179708:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6840 going to sleep for 11 seconds 00000100:00000001:0.0:1773941533.179713:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941533.179716:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941533.179804:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941533.179811:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a20bd00. 00000400:00000200:1.0:1773941533.179819:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941533.179826:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941533.179833:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d108334c8 00000400:00000010:1.0:1773941533.179836:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d108334c8. 00000100:00000001:1.0:1773941533.179841:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941533.179847:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10836a00 x1860112291483648/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941549 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941533.179860:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941533.179864:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d10836a00 x1860112291483648/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941549 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941533.179872:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941533.179874:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941533.180871:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941533.180884:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941533.180887:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941533.180890:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941533.180896:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941533.180901:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941533.180908:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941533.180913:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941533.180918:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941533.180923:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324605000 00000400:00000200:0.0:1773941533.180927:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941533.180934:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xce9 [1] + 224 00000800:00000001:0.0:1773941533.180942:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941533.180952:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941533.180954:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941533.180958:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941533.180983:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941533.180989:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10836a00 x1860112291483648/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941549 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941533.181001:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d10836a00 x1860112291483648/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941549 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941533.181022:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941533.181029:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941533.181041:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941533.181044:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941533.181052:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941533.181053:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941533.181056:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0014b790. 00000400:00000200:2.0:1773941533.181060:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833a18 00000400:00000010:2.0:1773941533.181062:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833a18. 00000100:00000001:2.0:1773941533.181064:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941533.181070:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10836a00 x1860112291483648/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941549 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941533.181082:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10836a00 x1860112291483648/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941549 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941533.181089:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941533.181099:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941533.181102:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941533.181105:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941533.181108:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941533.181111:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941533.181115:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941533.181122:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=34 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941533.181127:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941533.181130:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:2.0:1773941533.181134:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:2.0:1773941533.181136:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941533.181139:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:2.0:1773941533.181141:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1773941533.181145:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1773941533.181149:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1773941533.181152:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941533.181154:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941533.181155:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941533.181157:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:2.0:1773941533.181160:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d10836a00 x1860112291483648/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941549 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941533.181165:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941533.181166:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941533.181167:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941533.181170:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941533.181172:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:2.0:1773941533.181173:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941533.181174:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941533.181175:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941533.181176:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941533.181177:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941533.181178:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941533.181180:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1773941533.181182:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1773941533.181190:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941533.181193:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 254 00000400:00000200:2.0:1773941533.181197:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941533.181201:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:2.0:1773941533.181204:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941533.181207:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941533.181209:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941533.181211:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:2.0:1773941533.181213:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941533.181215:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941533.181217:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941533.181219:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941533.181221:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941533.181222:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:2.0:1773941533.181224:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941533.181225:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941533.181227:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941533.181228:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:2.0:1773941533.181231:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:2.0:1773941533.181234:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:2.0:1773941533.181235:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941533.181237:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941533.181240:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941533.181242:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941533.181243:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941533.181245:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941533.181248:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06506a00. 00000020:00000040:2.0:1773941533.181250:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:2.0:1773941533.181252:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941533.181254:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941533.181256:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941533.181257:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941533.181262:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941533.181266:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d035a7800. 00000100:00000001:2.0:1773941533.181269:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941533.181273:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d06506a00 x1860112291483776/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941533.181279:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941533.181281:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941533.181285:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6b603d8] to pc [ptlrpcd_rcv+-1] req@ffff988d06506a00 x1860112291483776/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941533.181303:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941533.181305:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:2.0:1773941533.181306:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941533.181309:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941533.181311:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941533.181312:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941533.181313:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6840 going to sleep for 1 seconds 00000100:00000001:2.0:1773941533.181314:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941533.181317:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941533.181321:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10836a00 x1860112291483648/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941549 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941533.181326:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941533.181329:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941533.181331:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941533.181333:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941533.181337:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10836a00 x1860112291483648/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941549 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941533.181341:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:2.0:1773941533.181344:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10836a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291483648:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941533.181348:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941533.181350:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941533.181352:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941533.181353:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:3.0:1773941533.181360:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1773941533.181361:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941533.181373:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06506a00 x1860112291483776/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941533.181373:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10836a00 x1860112291483648/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941549 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941533.181404:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941533.181406:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941533.181407:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941533.181413:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000010:2.0:1773941533.181413:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb6840. 02000000:00000001:3.0:1773941533.181416:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941533.181419:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 02000000:00000001:3.0:1773941533.181421:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941533.181423:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00100000:3.0:1773941533.181429:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291483776:192.168.204.114@tcp:250:ll_cfg_requeue.0 00010000:00000001:2.0:1773941533.181430:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:2.0:1773941533.181435:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000001:3.0:1773941533.181436:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000001:2.0:1773941533.181438:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:3.0:1773941533.181441:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:2.0:1773941533.181441:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00031e00 refcount=4 02000000:00000001:3.0:1773941533.181446:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941533.181446:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090855936 : -113743618695680 : ffff988d00031e00) 02000000:00000001:3.0:1773941533.181452:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:2.0:1773941533.181452:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090855936 : -113743618695680 : ffff988d00031e00) 00010000:00010000:2.0:1773941533.181458:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2975 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000010:3.0:1773941533.181459:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03522800. 02000000:00000001:3.0:1773941533.181465:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941533.181468:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000200:3.0:1773941533.181471:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:2.0:1773941533.181474:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2975 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:3.0:1773941533.181476:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00000400:00000010:3.0:1773941533.181482:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00010000:00000001:2.0:1773941533.181483:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000200:3.0:1773941533.181486:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291483776, portal 25 00010000:00010000:2.0:1773941533.181488:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2975 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941533.181491:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941533.181493:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584960 : -113743512966656 : ffff988d06506a00) 00010000:00000001:2.0:1773941533.181500:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000040:3.0:1773941533.181502:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06506a00 x1860112291483776/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941543 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1773941533.181503:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:2.0:1773941533.181507:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2975 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941533.181521:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:2.0:1773941533.181523:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000040:3.0:1773941533.181527:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00000001:2.0:1773941533.181527:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1773941533.181531:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182aca18. 00010000:00000001:2.0:1773941533.181532:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000100:00000200:3.0:1773941533.181534:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291483776, offset 0 00000400:00000200:3.0:1773941533.181539:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00010000:2.0:1773941533.181540:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2975 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941533.181546:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941533.181552:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00010000:2.0:1773941533.181553:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d00031e00 already unused, calling callback (ffffffffc12b4c20) 00000400:00000200:3.0:1773941533.181556:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 10000000:00000001:2.0:1773941533.181557:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:3.0:1773941533.181560:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 10000000:00010000:2.0:1773941533.181561:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2975 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941533.181575:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941533.181587:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:2.0:1773941533.181587:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:2.0:1773941533.181590:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:3.0:1773941533.181594:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[56, 57] 00000020:00000001:2.0:1773941533.181594:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941533.181597:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00031e00 refcount=5 00000400:00000200:3.0:1773941533.181601:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000020:00000001:2.0:1773941533.181602:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090855936 : -113743618695680 : ffff988d00031e00) 00000400:00000200:3.0:1773941533.181606:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [168:168:256:4294967295] 192.168.204.114@tcp LPNI seq info [58:58:8:4294967295] 00010000:00000001:2.0:1773941533.181606:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090855936 : -113743618695680 : ffff988d00031e00) 00010000:00000001:2.0:1773941533.181613:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:3.0:1773941533.181614:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:2.0:1773941533.181617:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2975 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941533.181627:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 10000000:00000001:2.0:1773941533.181629:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000800:00000200:3.0:1773941533.181633:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 10000000:00010000:2.0:1773941533.181633:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2975 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000010:3.0:1773941533.181640:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74800. 10000000:01000000:2.0:1773941533.181642:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:2.0:1773941533.181645:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000800:00000200:3.0:1773941533.181648:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 10000000:00000001:2.0:1773941533.181648:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941533.181651:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:2.0:1773941533.181654:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000800:00000200:3.0:1773941533.181657:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00010000:00000001:2.0:1773941533.181658:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1773941533.181661:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000800:00000200:3.0:1773941533.181663:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74800 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:2.0:1773941533.181664:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941533.181668:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941533.181670:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d00031e00 with handle 0x8e86767622e2975 from hash 00010000:00000001:2.0:1773941533.181672:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1773941533.181672:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941533.181677:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941533.181679:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941533.181682:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1773941533.181684:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000400:00000200:3.0:1773941533.181685:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:2.0:1773941533.181687:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000100:00000001:3.0:1773941533.181690:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294924777 00010000:00000001:2.0:1773941533.181691:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941533.181693:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941533.181697:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941533.181699:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941533.181702:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294924775 00010000:00010000:2.0:1773941533.181702:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2975 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941533.181712:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294924777 00010000:00000001:2.0:1773941533.181712:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941533.181714:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941533.181718:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:2.0:1773941533.181719:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000100:00000001:3.0:1773941533.181720:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294924775 00010000:00000001:2.0:1773941533.181721:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941533.181723:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941533.181727:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941533.181729:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00010000:2.0:1773941533.181730:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2975 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941533.181737:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000040:2.0:1773941533.181740:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d21ec1800 count: 0 00000100:00000001:3.0:1773941533.181743:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941533.181748:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1773941533.181752:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941533.181754:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941533.181756:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1773941533.181758:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:2.0:1773941533.181761:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10836a00 x1860112291483648/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941549 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941533.181762:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000800:00000200:1.0:1773941533.181765:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1773941533.181770:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000010:1.0:1773941533.181772:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74800. 00000100:00000001:3.0:1773941533.181773:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941533.181778:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941533.181779:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000400:00000200:1.0:1773941533.181781:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1773941533.181783:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941533.181784:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941533.181788:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d02ce7000. 00000400:00000200:1.0:1773941533.181792:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 02000000:00000001:2.0:1773941533.181793:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941533.181798:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000400:00000200:1.0:1773941533.181799:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182aca18 00000020:00000040:2.0:1773941533.181800:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941533.181804:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 00000400:00000010:1.0:1773941533.181804:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182aca18. 02000000:00000010:2.0:1773941533.181808:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065c4a00. 00000100:00000001:1.0:1773941533.181811:0:2393:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:2.0:1773941533.181814:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000200:1.0:1773941533.181818:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06506a00 x1860112291483776/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941543 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1773941533.181820:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941533.181823:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10836a00. 00000100:00000001:2.0:1773941533.181828:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941533.181830:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:2.0:1773941533.181835:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:2.0:1773941533.181838:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000100:00000001:1.0:1773941533.181839:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 10000000:00000001:2.0:1773941533.181841:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:2.0:1773941533.181846:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d05437280. 00000100:00000040:1.0:1773941533.181846:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06506a00 x1860112291483776/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941543 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1773941533.181853:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d08a4f400. 00000020:00000010:2.0:1773941533.181860:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d08a4fe00. 00000100:00000001:1.0:1773941533.181862:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000040:00000010:2.0:1773941533.181866:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d03356200. 00000100:00000001:1.0:1773941533.181866:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:2.0:1773941533.181873:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941533.181878:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d0381e800. 80000000:00000010:2.0:1773941533.181882:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d08bbb120. 00040000:00000010:2.0:1773941533.181887:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d0381e000. 20000000:00000010:2.0:1773941533.181891:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d0381f200. 10000000:00000040:2.0:1773941533.181902:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:2.0:1773941533.181904:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:2.0:1773941533.181908:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d08a4fe00. 00000040:00000010:2.0:1773941533.181913:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d03356200. 00000020:00000010:2.0:1773941533.181916:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941533.181919:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d0381e800. 80000000:00000010:2.0:1773941533.181921:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d08bbb120. 00040000:00000010:2.0:1773941533.181924:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d0381e000. 20000000:00000010:2.0:1773941533.181926:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d0381f200. 00000020:00000010:2.0:1773941533.181929:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d08a4f400. 10000000:00000010:2.0:1773941533.181931:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d05437280. 10000000:01000000:2.0:1773941533.181933:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:2.0:1773941533.181935:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:2.0:1773941533.181937:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:2.0:1773941533.181938:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:2.0:1773941533.181939:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:2.0:1773941533.181940:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941533.181942:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941533.183296:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941533.183304:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941533.183306:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941533.183309:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941533.183314:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941533.183319:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941533.183325:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941533.183329:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941533.183333:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941533.183336:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324605080 00000400:00000200:0.0:1773941533.183340:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941533.183345:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xcf9 [1] + 0 00000800:00000001:0.0:1773941533.183351:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941533.183358:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941533.183360:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941533.183364:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941533.183379:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941533.183388:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06506a00 x1860112291483776/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941543 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941533.183399:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d06506a00 x1860112291483776/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941543 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941533.183416:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941533.183421:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941533.183425:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941533.183444:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941533.183447:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941533.183451:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941533.183452:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941533.183455:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941533.183466:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941533.183467:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941533.183471:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941533.183476:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06506a00 x1860112291483776/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941543 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941533.183484:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06506a00 x1860112291483776/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941543 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941533.183489:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941533.183490:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941533.183491:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941533.183493:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941533.183494:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941533.183495:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941533.183497:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941533.183500:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=34 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941533.183502:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941533.183504:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941533.183505:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941533.183507:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941533.183508:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941533.183511:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941533.183513:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06506a00 x1860112291483776/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941543 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941533.183519:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941533.183520:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941533.183521:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941533.183523:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941533.183527:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941533.183529:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941533.183530:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941533.183531:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941533.183533:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941533.183534:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941533.183536:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941533.183544:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941533 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941533.183546:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941533.183552:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941533.183553:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941533.183555:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941533.183556:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941533.183558:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941533.183561:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941533.183577:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941533.183580:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941533.183583:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941533.183587:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941533.183588:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941533.183858:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941533.183862:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941533.183867:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06506a00 x1860112291483776/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941543 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941533.183875:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291483776:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941533.183877:0:14743:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941533.183879:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941533.183882:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941533.183885:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06506a00 x1860112291483776/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941543 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1773941533.183889:0:14743:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00000001:2.0:1773941533.183896:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00100000:3.0:1773941533.183897:0:14743:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:3.0:1773941533.183900:0:14743:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:2.0:1773941533.183900:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 10000000:01000000:3.0:1773941533.183904:0:14743:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000010:2.0:1773941533.183905:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03522800. 00000001:00000001:3.0:1773941533.183909:0:14743:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:2.0:1773941533.183909:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941533.183913:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:3.0:1773941533.183914:0:14743:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000040:2.0:1773941533.183914:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941533.183917:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:3.0:1773941533.183919:0:14743:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941533.183920:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d035a7800. 00000001:00000001:3.0:1773941533.183921:0:14743:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941533.183924:0:14743:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941533.183924:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941533.183927:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000001:00000001:3.0:1773941533.183928:0:14743:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941533.183931:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06506a00. 10000000:00000001:3.0:1773941533.183932:0:14743:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941533.183936:0:14743:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941533.183938:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941533.183940:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1773941533.183941:0:14743:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941533.183942:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941533.183946:0:14743:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941533.183948:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941533.183949:0:14743:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:2.0:1773941533.183952:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941533.183953:0:14743:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941533.183956:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1773941533.183957:0:14743:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:3.0:1773941533.183982:0:14743:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941533.183983:0:14743:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000100:00000001:2.0:1773941533.183984:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:00000040:3.0:1773941533.183985:0:14743:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:3.0:1773941533.184008:0:14743:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:3.0:1773941533.184009:0:14743:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941533.184011:0:14743:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:3.0:1773941533.184015:0:14743:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:3.0:1773941533.184018:0:14743:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941533.184020:0:14743:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941533.184021:0:14743:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941533.184023:0:14743:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941533.184024:0:14743:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:2.1:1773941533.186988:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d21ec1800. 00010000:00000010:2.1:1773941533.186994:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d00031e00. 00000020:00000001:1.0:1773941533.691108:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1773941533.691118:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941533.691150:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941534.459065:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294924777, at 4294924800 00000100:00000040:0.0:1773941534.459067:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294924777, at 4294924800 00000100:00000010:1.0:1773941534.459083:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580d80. 00000100:00000010:0.0:1773941534.459085:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580b40. 00000020:00000001:2.0:1773941534.715127:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941534.715134:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941534.715139:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:2.0:1773941535.035040:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:2.0:1773941535.035049:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941535.035054:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941535.035056:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941535.035082:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941535.035083:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:2.0:1773941535.035086:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:2.0:1773941535.035087:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:2.0:1773941535.035090:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:2.0:1773941535.035092:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:2.0:1773941535.035094:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:2.0:1773941535.035101:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06506a00. 00000020:00000040:2.0:1773941535.035107:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941535.035112:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941535.035115:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941535.035121:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941535.035128:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d0381f200. 00000100:00000001:2.0:1773941535.035135:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941535.035140:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:2.0:1773941535.035143:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:2.0:1773941535.035151:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d071eec00. 00010000:00000001:2.0:1773941535.035156:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:2.0:1773941535.035160:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d089de800. 00000020:00000001:2.0:1773941535.035164:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:2.0:1773941535.035166:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d089de800 with handle 0x8e86767622e297c to hash 00000020:00000001:2.0:1773941535.035168:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:2.0:1773941535.035170:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330235217920 : -113743474333696 : ffff988d089de800) 00010000:00000001:2.0:1773941535.035173:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330235217920 : -113743474333696 : ffff988d089de800) 00010000:00000001:2.0:1773941535.035176:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941535.035179:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941535.035182:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e297c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941535.035190:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e297c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941535.035201:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e297c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941535.035214:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:2.0:1773941535.035216:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:2.0:1773941535.035222:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d09350540. 00000100:00000001:2.0:1773941535.035224:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330245121344 : -113743464430272 : ffff988d09350540) 00000100:00000001:2.0:1773941535.035226:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941535.035228:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584960 : -113743512966656 : ffff988d06506a00) 00000100:00000001:2.0:1773941535.035232:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941535.035237:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941535.035238:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:2.0:1773941535.035240:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941535.035251:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06506a00 x1860112291483904/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941535.035257:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941535.035260:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941535.035263:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941535.035265:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941535.035268:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941535.035270:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291483904:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941535.035273:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941535.035279:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941535.035280:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941535.035282:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941535.035287:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a7800. 02000000:00000001:2.0:1773941535.035289:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941535.035297:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941535.035302:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000400:00000010:2.0:1773941535.035310:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000200:2.0:1773941535.035320:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291483904, portal 25 00000100:00000001:2.0:1773941535.035322:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941535.035324:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584960 : -113743512966656 : ffff988d06506a00) 00000100:00000040:2.0:1773941535.035330:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06506a00 x1860112291483904/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941551 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941535.035352:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941535.035355:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:2.0:1773941535.035358:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516330. 00000100:00000200:2.0:1773941535.035361:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291483904, offset 0 00000400:00000200:2.0:1773941535.035368:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941535.035376:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941535.035384:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941535.035389:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941535.035392:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941535.035396:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941535.035398:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941535.035401:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[58, 57] 00000400:00000200:2.0:1773941535.035404:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:2.0:1773941535.035408:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [169:169:256:4294967295] 192.168.204.124@tcp LPNI seq info [59:59:8:4294967295] 00000400:00000200:2.0:1773941535.035411:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:2.0:1773941535.035420:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:2.0:1773941535.035425:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:2.0:1773941535.035429:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0db000. 00000800:00000200:2.0:1773941535.035436:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:2.0:1773941535.035443:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:2.0:1773941535.035447:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0db000 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:2.0:1773941535.035465:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941535.035469:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941535.035472:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294926631 00000100:00000010:2.0:1773941535.035477:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d08bbb120. 00000100:00000001:2.0:1773941535.035487:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1773941535.035495:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294926631 00000100:00000010:2.0:1773941535.035498:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d08bbbc00. 00000100:00000001:2.0:1773941535.035502:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1773941535.035511:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941535.035514:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941535.035516:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941535.035518:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:2.0:1773941535.035521:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d09350540 going to sleep for 11 seconds 00000100:00000001:2.0:1773941535.035524:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941535.035541:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941535.035632:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941535.035637:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0db000. 00000400:00000200:1.0:1773941535.035643:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941535.035648:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941535.035653:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516330 00000400:00000010:1.0:1773941535.035655:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516330. 00000100:00000001:1.0:1773941535.035659:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941535.035663:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06506a00 x1860112291483904/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941551 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941535.035672:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941535.035674:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d06506a00 x1860112291483904/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941551 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941535.035679:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941535.035680:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941535.036591:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941535.036603:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941535.036605:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941535.036607:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941535.036611:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941535.036615:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941535.036628:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941535.036631:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941535.036635:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941535.036638:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324605100 00000400:00000200:0.0:1773941535.036642:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941535.036646:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xd01 [1] + 224 00000800:00000001:0.0:1773941535.036652:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941535.036668:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941535.036670:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941535.036673:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941535.036678:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941535.036683:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06506a00 x1860112291483904/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941551 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941535.036692:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d06506a00 x1860112291483904/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941551 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941535.036705:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941535.036710:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941535.036713:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941535.036742:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941535.036750:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941535.036752:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941535.036754:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941535.036758:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941535.036760:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941535.036762:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941535.036766:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06506a00 x1860112291483904/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941551 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941535.036775:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06506a00 x1860112291483904/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941551 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941535.036781:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941535.036782:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941535.036784:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941535.036787:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941535.036789:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941535.036790:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941535.036792:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941535.036798:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=36 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941535.036800:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941535.036803:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:2.0:1773941535.036806:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:2.0:1773941535.036809:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941535.036811:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:2.0:1773941535.036814:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1773941535.036817:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1773941535.036820:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1773941535.036830:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941535.036832:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941535.036833:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941535.036835:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:2.0:1773941535.036838:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d06506a00 x1860112291483904/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941551 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941535.036843:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941535.036845:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941535.036846:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941535.036849:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941535.036851:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:2.0:1773941535.036852:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941535.036854:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941535.036854:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941535.036855:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941535.036856:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941535.036857:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941535.036860:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1773941535.036861:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1773941535.036863:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941535.036865:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 256 00000400:00000200:2.0:1773941535.036868:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941535.036872:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:2.0:1773941535.036875:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941535.036878:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941535.036880:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941535.036882:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:2.0:1773941535.036885:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941535.036887:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941535.036888:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941535.036890:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941535.036892:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941535.036893:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:2.0:1773941535.036899:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941535.036900:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941535.036902:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941535.036903:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:2.0:1773941535.036906:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:2.0:1773941535.036908:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:2.0:1773941535.036910:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941535.036912:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941535.036915:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941535.036916:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941535.036918:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941535.036920:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941535.036922:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06505880. 00000020:00000040:2.0:1773941535.036924:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:2.0:1773941535.036926:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941535.036928:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941535.036930:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941535.036932:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941535.036933:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941535.036937:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d035a6400. 00000100:00000001:2.0:1773941535.036940:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941535.036944:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d06505880 x1860112291484032/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941535.036950:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941535.036952:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941535.036956:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e62ba1de] to pc [ptlrpcd_rcv+-1] req@ffff988d06505880 x1860112291484032/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941535.036991:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941535.036993:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:2.0:1773941535.036994:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941535.036996:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941535.036998:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941535.037003:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941535.037005:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d09350540 going to sleep for 1 seconds 00000100:00000001:2.0:1773941535.037007:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941535.037009:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941535.037014:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06506a00 x1860112291483904/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941551 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941535.037020:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941535.037022:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941535.037024:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941535.037024:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941535.037027:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941535.037029:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941535.037033:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941535.037034:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06506a00 x1860112291483904/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941551 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941535.037043:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06505880 x1860112291484032/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941535.037044:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291483904:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941535.037053:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941535.037057:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941535.037059:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941535.037060:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:3.0:1773941535.037063:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941535.037063:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1773941535.037068:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:2.0:1773941535.037069:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06506a00 x1860112291483904/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941551 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1773941535.037072:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941535.037077:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941535.037082:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941535.037084:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06505880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291484032:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000010:2.0:1773941535.037085:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d09350540. 00000100:00000001:2.0:1773941535.037090:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:3.0:1773941535.037091:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1773941535.037093:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:3.0:1773941535.037095:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00000001:2.0:1773941535.037096:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 02000000:00000001:3.0:1773941535.037098:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941535.037100:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 02000000:00000001:3.0:1773941535.037101:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:2.0:1773941535.037102:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941535.037105:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089de800 refcount=4 02000000:00000010:3.0:1773941535.037107:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03523000. 00000020:00000001:2.0:1773941535.037107:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235217920 : -113743474333696 : ffff988d089de800) 02000000:00000001:3.0:1773941535.037112:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941535.037113:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235217920 : -113743474333696 : ffff988d089de800) 00010000:00010000:2.0:1773941535.037119:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e297c lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941535.037120:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941535.037123:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00000400:00000010:3.0:1773941535.037130:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00010000:00000001:2.0:1773941535.037134:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000200:3.0:1773941535.037136:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291484032, portal 25 00000100:00000001:3.0:1773941535.037140:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00010000:2.0:1773941535.037141:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e297c lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941535.037142:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196580480 : -113743512971136 : ffff988d06505880) 00010000:00000001:2.0:1773941535.037151:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000040:3.0:1773941535.037152:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06505880 x1860112291484032/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941545 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:2.0:1773941535.037156:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e297c lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941535.037170:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941535.037172:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:2.0:1773941535.037174:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e297c lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941535.037177:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941535.037179:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941535.037181:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:2.0:1773941535.037183:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e297c lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941535.037187:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d089de800 already unused, calling callback (ffffffffc12b4c20) 10000000:00000001:2.0:1773941535.037190:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:2.0:1773941535.037192:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e297c lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941535.037208:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000100:00000001:3.0:1773941535.037210:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:2.0:1773941535.037210:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:2.0:1773941535.037213:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941535.037216:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089de800 refcount=5 00000100:00000040:3.0:1773941535.037217:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000020:00000001:2.0:1773941535.037220:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235217920 : -113743474333696 : ffff988d089de800) 00000400:00000010:3.0:1773941535.037222:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac330. 00010000:00000001:2.0:1773941535.037223:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235217920 : -113743474333696 : ffff988d089de800) 00000100:00000200:3.0:1773941535.037227:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291484032, offset 0 00010000:00000001:2.0:1773941535.037228:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:3.0:1773941535.037233:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00010000:2.0:1773941535.037238:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e297c lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941535.037241:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941535.037248:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 10000000:00000001:2.0:1773941535.037249:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:3.0:1773941535.037253:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 10000000:00010000:2.0:1773941535.037254:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e297c lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941535.037258:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941535.037262:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 10000000:01000000:2.0:1773941535.037265:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:3.0:1773941535.037267:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 10000000:00000040:2.0:1773941535.037271:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:2.0:1773941535.037274:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941535.037275:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[58, 59] 00010000:00010000:2.0:1773941535.037278:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000400:00000200:3.0:1773941535.037281:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00010000:00000001:2.0:1773941535.037282:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:3.0:1773941535.037286:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [170:170:256:4294967295] 192.168.204.114@tcp LPNI seq info [60:60:8:4294967295] 00010000:00000001:2.0:1773941535.037287:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1773941535.037290:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1773941535.037293:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:3.0:1773941535.037295:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:2.0:1773941535.037295:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941535.037301:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d089de800 with handle 0x8e86767622e297c from hash 00010000:00000001:2.0:1773941535.037304:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1773941535.037308:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941535.037311:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:3.0:1773941535.037313:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00010000:00000001:2.0:1773941535.037314:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1773941535.037318:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:2.0:1773941535.037321:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000800:00000200:3.0:1773941535.037325:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00010000:00000001:2.0:1773941535.037327:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941535.037330:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941535.037332:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1773941535.037333:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74c00. 10000000:00000001:2.0:1773941535.037335:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941535.037340:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e297c lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941535.037342:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00010000:00000001:2.0:1773941535.037353:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941535.037357:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000200:3.0:1773941535.037358:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00010000:00000001:2.0:1773941535.037360:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:2.0:1773941535.037364:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000800:00000200:3.0:1773941535.037365:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74c00 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:2.0:1773941535.037366:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941535.037369:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941535.037375:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1773941535.037378:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e297c lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941535.037382:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941535.037390:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000040:2.0:1773941535.037391:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d071eec00 count: 0 00000100:00000001:3.0:1773941535.037394:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294926633 00000100:00000001:3.0:1773941535.037403:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294926631 00000100:00000001:3.0:1773941535.037408:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294926633 00010000:00000001:2.0:1773941535.037409:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:3.0:1773941535.037411:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294926631 00010000:00000001:2.0:1773941535.037413:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941535.037416:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941535.037419:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941535.037419:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941535.037427:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06506a00 x1860112291483904/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941551 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941535.037429:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941535.037434:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941535.037443:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1773941535.037444:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941535.037447:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 02000000:00000001:2.0:1773941535.037449:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1773941535.037450:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941535.037453:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000800:00000200:1.0:1773941535.037454:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1773941535.037455:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941535.037455:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a7800. 00000100:00000001:3.0:1773941535.037460:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1773941535.037460:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74c00. 02000000:00000001:2.0:1773941535.037461:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941535.037465:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000400:00000200:1.0:1773941535.037467:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000040:2.0:1773941535.037469:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941535.037471:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 00000400:00000200:1.0:1773941535.037472:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 02000000:00000010:2.0:1773941535.037475:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d0381f200. 00000400:00000200:1.0:1773941535.037476:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac330 00000400:00000010:1.0:1773941535.037478:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac330. 02000000:00000001:2.0:1773941535.037479:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941535.037481:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1773941535.037482:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000010:2.0:1773941535.037483:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06506a00. 00000100:00000200:1.0:1773941535.037486:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06505880 x1860112291484032/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941545 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941535.037487:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941535.037489:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:2.0:1773941535.037491:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:2.0:1773941535.037494:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000100:00000001:1.0:1773941535.037495:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 10000000:00000001:2.0:1773941535.037497:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000100:00000040:1.0:1773941535.037499:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06505880 x1860112291484032/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941545 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000010:2.0:1773941535.037502:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d05437080. 00000100:00000001:1.0:1773941535.037506:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1773941535.037507:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d0381f200. 00000100:00000001:1.0:1773941535.037508:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:2.0:1773941535.037512:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d0381e000. 00000040:00000010:2.0:1773941535.037517:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d0381e800. 00000020:00000010:2.0:1773941535.037523:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941535.037548:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d03356200. 80000000:00000010:2.0:1773941535.037553:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d08bbb900. 00040000:00000010:2.0:1773941535.037559:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d08a4f400. 20000000:00000010:2.0:1773941535.037564:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d08a4fe00. 10000000:00000040:2.0:1773941535.037573:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:2.0:1773941535.037575:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:2.0:1773941535.037580:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d0381e000. 00000040:00000010:2.0:1773941535.037589:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d0381e800. 00000020:00000010:2.0:1773941535.037592:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941535.037595:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d03356200. 80000000:00000010:2.0:1773941535.037598:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d08bbb900. 00040000:00000010:2.0:1773941535.037601:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d08a4f400. 20000000:00000010:2.0:1773941535.037605:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d08a4fe00. 00000020:00000010:2.0:1773941535.037607:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0381f200. 10000000:00000010:2.0:1773941535.037609:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d05437080. 10000000:01000000:2.0:1773941535.037612:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:2.0:1773941535.037614:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:2.0:1773941535.037616:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:2.0:1773941535.037618:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:2.0:1773941535.037619:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:2.0:1773941535.037620:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941535.037622:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941535.040217:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941535.040224:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941535.040226:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941535.040228:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941535.040232:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941535.040235:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941535.040240:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941535.040242:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941535.040245:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941535.040248:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324605180 00000400:00000200:0.0:1773941535.040250:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941535.040254:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xd09 [1] + 0 00000800:00000001:0.0:1773941535.040259:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941535.040270:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941535.040272:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941535.040275:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941535.040278:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941535.040282:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06505880 x1860112291484032/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941545 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941535.040290:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d06505880 x1860112291484032/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941545 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941535.040305:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941535.040311:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941535.040314:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941535.040339:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941535.040344:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941535.040349:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941535.040351:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941535.040353:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941535.040359:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941535.040361:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941535.040364:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941535.040369:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06505880 x1860112291484032/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941545 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941535.040378:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06505880 x1860112291484032/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941545 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941535.040383:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941535.040384:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941535.040386:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941535.040387:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941535.040389:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941535.040390:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941535.040391:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941535.040395:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=36 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941535.040398:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941535.040400:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941535.040401:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941535.040404:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941535.040405:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941535.040407:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941535.040410:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06505880 x1860112291484032/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941545 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941535.040426:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941535.040428:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941535.040429:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941535.040431:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941535.040435:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941535.040437:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941535.040438:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941535.040440:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941535.040442:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941535.040444:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941535.040446:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941535.040449:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941535 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941535.040451:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941535.040457:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941535.040458:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941535.040460:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941535.040461:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941535.040463:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941535.040466:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941535.040468:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941535.040470:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941535.040473:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941535.040476:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941535.040477:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941535.040722:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941535.040725:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941535.040731:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06505880 x1860112291484032/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941545 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941535.040739:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06505880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291484032:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941535.040741:0:14745:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941535.040743:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941535.040747:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941535.040750:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06505880 x1860112291484032/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941545 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1773941535.040756:0:14745:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00000001:2.0:1773941535.040758:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941535.040762:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00100000:3.0:1773941535.040764:0:14745:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000010:2.0:1773941535.040765:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03523000. 00000100:00000001:3.0:1773941535.040770:0:14745:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:2.0:1773941535.040772:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 10000000:01000000:3.0:1773941535.040774:0:14745:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:3.0:1773941535.040778:0:14745:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:2.0:1773941535.040780:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:3.0:1773941535.040785:0:14745:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000040:2.0:1773941535.040785:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941535.040789:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941535.040791:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d035a6400. 02000000:00000001:2.0:1773941535.040794:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941535.040798:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000001:00000001:3.0:1773941535.040799:0:14745:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941535.040800:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06505880. 00000001:00000001:3.0:1773941535.040805:0:14745:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:2.0:1773941535.040807:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000001:00000001:3.0:1773941535.040809:0:14745:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941535.040811:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:3.0:1773941535.040813:0:14745:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941535.040816:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:3.0:1773941535.040819:0:14745:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941535.040823:0:14745:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941535.040824:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941535.040828:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1773941535.040829:0:14745:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941535.040832:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941535.040835:0:14745:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941535.040837:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941535.040840:0:14745:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941535.040844:0:14745:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1773941535.040847:0:14745:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:3.0:1773941535.040850:0:14745:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941535.040852:0:14745:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:3.0:1773941535.040854:0:14745:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:3.0:1773941535.040870:0:14745:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:3.0:1773941535.040871:0:14745:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941535.040872:0:14745:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:3.0:1773941535.040875:0:14745:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:3.0:1773941535.040877:0:14745:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941535.040879:0:14745:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941535.040881:0:14745:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941535.040882:0:14745:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941535.040883:0:14745:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:2.1:1773941535.043015:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d071eec00. 00010000:00000010:2.1:1773941535.043031:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d089de800. 00000020:00000001:0.0:1773941535.739122:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941535.739129:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941535.739133:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1773941535.995061:0:461:0:(pinger.c:229:ptlrpc_pinger_process_import()) c1c1ff21-5f68-4bf6-bdc4-258644b24465->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1773941535.995072:0:461:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:3.0:1773941535.995081:0:461:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d111f0a80. 00000020:00000040:3.0:1773941535.995088:0:461:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941535.995093:0:461:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941535.995096:0:461:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941535.995102:0:461:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941535.995111:0:461:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff988d08f74c00. 00000100:00000001:3.0:1773941535.995118:0:461:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941535.995128:0:461:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging c1c1ff21-5f68-4bf6-bdc4-258644b24465->MGS req@ffff988d111f0a80 x1860112291484160/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1773941535.995137:0:461:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941535.995154:0:461:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941535.995160:0:461:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b933c3a] to pc [ptlrpcd_01_00+0] req@ffff988d111f0a80 x1860112291484160/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941535.995183:0:461:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941535.995186:0:461:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (264) 00000100:00000001:3.0:1773941535.995194:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:3.0:1773941535.995197:0:2404:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:3.0:1773941535.995199:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941535.995204:0:2404:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941535.995206:0:2404:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941535.995207:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941535.995209:0:2404:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941535.995210:0:2404:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941535.995215:0:2404:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d111f0a80 x1860112291484160/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941535.995220:0:2404:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941535.995222:0:2404:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941535.995225:0:2404:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941535.995227:0:2404:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941535.995229:0:2404:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941535.995232:0:2404:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d111f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2404:1860112291484160:192.168.204.114@tcp:400:kworker.0 00000100:00000001:3.0:1773941535.995236:0:2404:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941535.995240:0:2404:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941535.995242:0:2404:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941535.995243:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1773941535.995244:0:2404:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1773941535.995248:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941535.995250:0:2404:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff988d1a1c8e00. 02000000:00000001:3.0:1773941535.995253:0:2404:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941535.995260:0:2404:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941535.995263:0:2404:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00000400:00000010:3.0:1773941535.995270:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00000100:00000200:3.0:1773941535.995276:0:2404:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112291484160, portal 25 00000100:00000001:3.0:1773941535.995277:0:2404:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941535.995279:0:2404:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330377898624 : -113743331652992 : ffff988d111f0a80) 00000100:00000040:3.0:1773941535.995283:0:2404:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d111f0a80 x1860112291484160/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941551 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941535.995293:0:2404:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941535.995295:0:2404:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:3.0:1773941535.995297:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acc38. 00000100:00000200:3.0:1773941535.995300:0:2404:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1860112291484160, offset 0 00000400:00000200:3.0:1773941535.995305:0:2404:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941535.995312:0:2404:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941535.995318:0:2404:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941535.995322:0:2404:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941535.995325:0:2404:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941535.995329:0:2404:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941535.995331:0:2404:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941535.995334:0:2404:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[60, 59] 00000400:00000200:3.0:1773941535.995347:0:2404:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:3.0:1773941535.995350:0:2404:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [171:171:256:4294967295] 192.168.204.124@tcp LPNI seq info [61:61:8:4294967295] 00000400:00000200:3.0:1773941535.995353:0:2404:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:3.0:1773941535.995361:0:2404:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:3.0:1773941535.995365:0:2404:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:3.0:1773941535.995369:0:2404:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74800. 00000800:00000200:3.0:1773941535.995374:0:2404:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:3.0:1773941535.995380:0:2404:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:3.0:1773941535.995384:0:2404:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74800 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1773941535.995395:0:2404:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941535.995400:0:2404:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:3.0:1773941535.995414:0:2404:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294927591 00000100:00000001:3.0:1773941535.995421:0:2404:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294926633 00000100:00000001:3.0:1773941535.995425:0:2404:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294927591 00000100:00000001:3.0:1773941535.995428:0:2404:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294926633 00000100:00000001:3.0:1773941535.995432:0:2404:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941535.995434:0:2404:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941535.995442:0:2404:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941535.995445:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941535.995548:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941535.995568:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74800. 00000400:00000200:1.0:1773941535.995571:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941535.995576:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941535.995580:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acc38 00000400:00000010:1.0:1773941535.995582:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acc38. 00000100:00000001:1.0:1773941535.995585:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941535.995588:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d111f0a80 x1860112291484160/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941551 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941535.995595:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941535.995598:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d111f0a80 x1860112291484160/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941551 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941535.995603:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941535.995604:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941535.996835:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941535.996848:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941535.996850:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941535.996852:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941535.996856:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941535.996860:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941535.996866:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941535.996869:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941535.996872:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941535.996875:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324605200 00000400:00000200:0.0:1773941535.996878:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941535.996883:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xd11 [1] + 224 00000800:00000001:0.0:1773941535.996890:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941535.996897:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941535.996899:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941535.996901:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941535.996904:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941535.996909:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d111f0a80 x1860112291484160/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941551 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941535.996928:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff988d111f0a80 x1860112291484160/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941551 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941535.996944:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941535.996950:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941535.996953:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941535.997022:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941535.997029:0:2404:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941535.997038:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941535.997040:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941535.997043:0:2404:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941535.997051:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941535.997053:0:2404:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941535.997056:0:2404:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941535.997065:0:2404:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d111f0a80 x1860112291484160/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941551 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941535.997079:0:2404:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d111f0a80 x1860112291484160/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941551 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941535.997090:0:2404:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941535.997092:0:2404:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941535.997095:0:2404:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941535.997099:0:2404:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941535.997103:0:2404:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941535.997106:0:2404:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941535.997110:0:2404:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941535.997119:0:2404:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=36 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941535.997125:0:2404:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=0 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941535.997129:0:2404:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941535.997132:0:2404:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941535.997137:0:2404:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:3.0:1773941535.997140:0:2404:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:3.0:1773941535.997146:0:2404:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1773941535.997152:0:2404:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1773941536.002272:0:2404:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:3.0:1773941536.002276:0:2404:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941536.002278:0:2404:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:3.0:1773941536.002281:0:2404:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:3.0:1773941536.002304:0:2404:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d111f0a80 x1860112291484160/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941551 ref 1 fl Rpc:RNQU/200/0 rc 0/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941536.002315:0:2404:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941536.002317:0:2404:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941536.002336:0:2404:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941536.002340:0:2404:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1773941536.002343:0:2404:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:3.0:1773941536.002344:0:2404:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941536.002347:0:2404:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941536.002348:0:2404:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941536.002349:0:2404:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:3.0:1773941536.002350:0:2404:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941536.002351:0:2404:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941536.002353:0:2404:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1773941536.002355:0:2404:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1773941536.002358:0:2404:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1773941536.002361:0:2404:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 258 00000400:00000200:3.0:1773941536.002367:0:2404:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941536.002372:0:2404:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:3.0:1773941536.002376:0:2404:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:3.0:1773941536.002384:0:2404:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941536.002387:0:2404:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:3.0:1773941536.002390:0:2404:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:3.0:1773941536.002394:0:2404:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941536.002396:0:2404:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941536.002399:0:2404:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941536.002402:0:2404:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941536.002404:0:2404:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941536.002406:0:2404:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:3.0:1773941536.002409:0:2404:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:3.0:1773941536.002410:0:2404:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941536.002411:0:2404:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:3.0:1773941536.002413:0:2404:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:3.0:1773941536.002439:0:2404:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:3.0:1773941536.002443:0:2404:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:3.0:1773941536.002446:0:2404:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941536.002449:0:2404:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1773941536.002453:0:2404:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941536.002455:0:2404:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941536.002458:0:2404:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:3.0:1773941536.002460:0:2404:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941536.002465:0:2404:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d111f3480. 00000020:00000040:3.0:1773941536.002469:0:2404:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:3.0:1773941536.002473:0:2404:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1773941536.002475:0:2404:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941536.002478:0:2404:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941536.002481:0:2404:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941536.002487:0:2404:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941536.002495:0:2404:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d03520c00. 00000100:00000001:3.0:1773941536.002503:0:2404:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941536.002533:0:2404:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d111f3480 x1860112291484288/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1773941536.002563:0:2404:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941536.002569:0:2404:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941536.002576:0:2404:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000097ce2b1d] to pc [ptlrpcd_rcv+-1] req@ffff988d111f3480 x1860112291484288/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941536.002605:0:2404:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941536.002609:0:2404:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:3.0:1773941536.002628:0:2404:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:3.0:1773941536.002636:0:2404:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d111f0a80 x1860112291484160/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941551 ref 1 fl Rpc:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941536.002649:0:2404:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941536.002653:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941536.002657:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941536.002674:0:2404:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d111f0a80 x1860112291484160/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941551 ref 1 fl Interpret:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941536.002677:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941536.002682:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941536.002687:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00100000:3.0:1773941536.002699:0:2404:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d111f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2404:1860112291484160:192.168.204.114@tcp:400:kworker.0 00000100:00000040:2.0:1773941536.002702:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d111f3480 x1860112291484288/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941536.002728:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941536.002729:0:2404:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941536.002733:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941536.002735:0:2404:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1773941536.002742:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:3.0:1773941536.002748:0:2404:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d111f0a80 x1860112291484160/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941551 ref 1 fl Complete:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1773941536.002749:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941536.002760:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941536.002774:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d111f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291484288:192.168.204.114@tcp:250:ptlrpcd_01_01.0 00000100:00000001:3.0:1773941536.002776:0:2404:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941536.002782:0:2404:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1773941536.002784:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000010:3.0:1773941536.002788:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff988d1a1c8e00. 02000000:00000001:2.0:1773941536.002790:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941536.002795:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941536.002796:0:2404:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:2.0:1773941536.002799:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:3.0:1773941536.002801:0:2404:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941536.002805:0:2404:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 02000000:00000010:2.0:1773941536.002808:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a6400. 00000020:00000001:3.0:1773941536.002809:0:2404:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941536.002814:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff988d08f74c00. 02000000:00000001:2.0:1773941536.002815:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941536.002820:0:2404:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941536.002824:0:2404:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000400:00000200:2.0:1773941536.002825:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000010:3.0:1773941536.002828:0:2404:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d111f0a80. 00000100:00000001:3.0:1773941536.002834:0:2404:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000400:00000010:2.0:1773941536.002836:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000100:00000001:3.0:1773941536.002837:0:2404:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941536.002843:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1773941536.002849:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000001:3.0:1773941536.002850:0:2404:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941536.002853:0:2404:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941536.002855:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941536.002857:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1773941536.002867:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291484288, portal 25 00000100:00000001:2.0:1773941536.002871:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941536.002872:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330377909376 : -113743331642240 : ffff988d111f3480) 00000100:00000040:2.0:1773941536.002881:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d111f3480 x1860112291484288/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941546 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941536.002890:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941536.002893:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:2.0:1773941536.002896:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516cc0. 00000100:00000200:2.0:1773941536.002898:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291484288, offset 0 00000400:00000200:2.0:1773941536.002903:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941536.002909:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941536.002914:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941536.002918:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941536.002921:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941536.002924:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941536.002927:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941536.002930:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[60, 61] 00000400:00000200:2.0:1773941536.002934:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:2.0:1773941536.002938:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [172:172:256:4294967295] 192.168.204.114@tcp LPNI seq info [62:62:8:4294967295] 00000400:00000200:2.0:1773941536.002942:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941536.002950:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:2.0:1773941536.002953:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:2.0:1773941536.002957:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0dbd00. 00000800:00000200:2.0:1773941536.002993:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:2.0:1773941536.003001:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:2.0:1773941536.003004:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0dbd00 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:2.0:1773941536.003014:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941536.003020:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941536.003022:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294927599 00000100:00000001:2.0:1773941536.003027:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294927591 00000100:00000001:2.0:1773941536.003037:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294927599 00000100:00000001:2.0:1773941536.003039:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294927591 00000100:00000001:2.0:1773941536.003041:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941536.003043:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941536.003045:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941536.003047:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941536.003050:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941536.003052:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941536.003053:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941536.003054:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941536.003055:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941536.003057:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941536.003192:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941536.003195:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0dbd00. 00000400:00000200:1.0:1773941536.003199:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941536.003203:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941536.003206:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516cc0 00000400:00000010:1.0:1773941536.003208:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516cc0. 00000100:00000001:1.0:1773941536.003210:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941536.003214:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d111f3480 x1860112291484288/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941546 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941536.003221:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941536.003224:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d111f3480 x1860112291484288/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941546 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941536.003228:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941536.003230:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941536.008703:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941536.008711:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941536.008713:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941536.008715:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941536.008719:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941536.008723:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941536.008727:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941536.008730:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941536.008733:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941536.008735:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324605280 00000400:00000200:0.0:1773941536.008738:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941536.008741:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xd19 [1] + 0 00000800:00000001:0.0:1773941536.008745:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941536.008751:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941536.008753:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941536.008755:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941536.008758:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941536.008763:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d111f3480 x1860112291484288/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941546 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941536.008771:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d111f3480 x1860112291484288/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941546 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941536.008787:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941536.008792:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941536.008794:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941536.008822:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941536.008824:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941536.008828:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941536.008830:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941536.008831:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941536.008834:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941536.008835:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941536.008837:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941536.008842:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d111f3480 x1860112291484288/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941546 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941536.008849:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d111f3480 x1860112291484288/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941546 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941536.008855:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941536.008856:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941536.008858:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941536.008859:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941536.008861:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941536.008863:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941536.008864:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941536.008870:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=37 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941536.008872:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941536.008874:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941536.008875:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941536.008884:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941536.008886:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941536.008889:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941536.008892:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d111f3480 x1860112291484288/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941546 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941536.008897:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941536.008899:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941536.008900:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941536.008902:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941536.008907:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941536.008909:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941536.008910:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941536.008912:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941536.008914:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941536.008916:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941536.008917:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:2.0:1773941536.008920:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941536 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941536.008922:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941536.008927:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:2.0:1773941536.008929:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941536.008931:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941536.008933:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941536.008934:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941536.008936:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:2.0:1773941536.008938:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941536.008940:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941536.008942:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941536.008945:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:2.0:1773941536.008947:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941536.009167:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941536.009175:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941536.009189:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d111f3480 x1860112291484288/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941546 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941536.009191:0:14746:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00080000:2.0:1773941536.009203:0:14746:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:2.0:1773941536.009208:0:14746:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00100000:3.0:1773941536.009218:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d111f3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291484288:192.168.204.114@tcp:250:ptlrpcd_01_01.0 00000100:00000001:3.0:1773941536.009233:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941536.009237:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941536.009242:0:14746:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941536.009244:0:14746:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:2.0:1773941536.009247:0:14746:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941536.009250:0:14746:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941536.009255:0:14746:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941536.009257:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d111f3480 x1860112291484288/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941546 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000001:00000001:2.0:1773941536.009260:0:14746:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941536.009263:0:14746:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941536.009270:0:14746:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941536.009278:0:14746:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941536.009287:0:14746:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941536.009295:0:14746:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:3.0:1773941536.009298:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1773941536.009299:0:14746:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 02000000:00000001:3.0:1773941536.009305:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1773941536.009305:0:14746:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:2.0:1773941536.009307:0:14746:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941536.009311:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a6400. 00000100:00080000:2.0:1773941536.009312:0:14746:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941536.009315:0:14746:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941536.009321:0:14746:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:2.0:1773941536.009323:0:14746:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 02000000:00000001:3.0:1773941536.009324:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941536.009328:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941536.009331:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941536.009335:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941536.009337:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d03520c00. 00000100:00080000:2.0:1773941536.009341:0:14746:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 02000000:00000001:3.0:1773941536.009343:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 10000000:00000001:2.0:1773941536.009346:0:14746:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941536.009347:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941536.009351:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d111f3480. 00000100:00000001:2.0:1773941536.009352:0:14746:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941536.009359:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941536.009363:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:02000000:2.0:1773941536.009365:0:14746:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:3.0:1773941536.009370:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941536.009381:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941536.009385:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941536.009393:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941536.009395:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941536.012878:0:14746:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941536.012883:0:14746:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941536.012885:0:14746:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941536.012889:0:14746:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941536.012890:0:14746:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941536.243045:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1773941536.243068:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941536.243073:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941536.243076:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941536.243077:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941536.243079:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1773941536.243082:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:0.0:1773941536.243085:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:0.0:1773941536.243087:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:0.0:1773941536.243090:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1773941536.243092:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:0.0:1773941536.243101:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10836300. 00000020:00000040:0.0:1773941536.243107:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941536.243112:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941536.243115:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941536.243120:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941536.243128:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065c5c00. 00000100:00000001:0.0:1773941536.243135:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941536.243140:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1773941536.243143:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1773941536.243152:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d21ec1800. 00010000:00000001:0.0:1773941536.243157:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1773941536.243160:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d00031e00. 00000020:00000001:0.0:1773941536.243165:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:0.0:1773941536.243168:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d00031e00 with handle 0x8e86767622e2983 to hash 00000020:00000001:0.0:1773941536.243170:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:0.0:1773941536.243172:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330090855936 : -113743618695680 : ffff988d00031e00) 00010000:00000001:0.0:1773941536.243175:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330090855936 : -113743618695680 : ffff988d00031e00) 00010000:00000001:0.0:1773941536.243179:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941536.243183:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941536.243187:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2983 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941536.243199:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2983 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941536.243222:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2983 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941536.243228:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1773941536.243231:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941536.243237:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb66c0. 00000100:00000001:0.0:1773941536.243239:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330139633344 : -113743569918272 : ffff988d02eb66c0) 00000100:00000001:0.0:1773941536.243242:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941536.243244:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367697664 : -113743341853952 : ffff988d10836300) 00000100:00000001:0.0:1773941536.243250:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941536.243254:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941536.243257:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941536.243259:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941536.243269:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10836300 x1860112291484416/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941536.243278:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941536.243280:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941536.243284:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941536.243287:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941536.243290:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941536.243293:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10836300 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291484416:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941536.243298:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941536.243303:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941536.243306:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941536.243308:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941536.243313:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d02ce5800. 02000000:00000001:0.0:1773941536.243315:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941536.243322:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941536.243326:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0014b420. 00000400:00000010:0.0:1773941536.243332:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833440. 00000100:00000200:0.0:1773941536.243339:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291484416, portal 25 00000100:00000001:0.0:1773941536.243341:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941536.243347:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367697664 : -113743341853952 : ffff988d10836300) 00000100:00000040:0.0:1773941536.243354:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10836300 x1860112291484416/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941552 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941536.243362:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941536.243365:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941536.243367:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833880. 00000100:00000200:0.0:1773941536.243371:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291484416, offset 0 00000400:00000200:0.0:1773941536.243378:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941536.243386:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941536.243393:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941536.243398:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941536.243403:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941536.243407:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941536.243410:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941536.243413:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[62, 61] 00000400:00000200:0.0:1773941536.243418:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:0.0:1773941536.243422:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [173:173:256:4294967295] 192.168.204.124@tcp LPNI seq info [63:63:8:4294967295] 00000400:00000200:0.0:1773941536.243427:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941536.243437:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:0.0:1773941536.243445:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:0.0:1773941536.243450:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a20bd00. 00000800:00000200:0.0:1773941536.243456:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:0.0:1773941536.243465:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:0.0:1773941536.243469:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a20bd00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1773941536.243488:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941536.243492:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941536.243495:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294927839 00000100:00000001:0.0:1773941536.243502:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294927599 00000100:00000001:0.0:1773941536.243520:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294927839 00000100:00000001:0.0:1773941536.243524:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294927599 00000100:00000001:0.0:1773941536.243533:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941536.243537:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941536.243540:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941536.243542:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1773941536.243546:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb66c0 going to sleep for 11 seconds 00000100:00000001:0.0:1773941536.243549:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941536.243552:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941536.243643:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941536.243648:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a20bd00. 00000400:00000200:1.0:1773941536.243653:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941536.243657:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941536.243662:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833880 00000400:00000010:1.0:1773941536.243664:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833880. 00000100:00000001:1.0:1773941536.243673:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941536.243678:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10836300 x1860112291484416/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941552 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941536.243686:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941536.243689:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d10836300 x1860112291484416/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941552 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941536.243694:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941536.243696:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941536.244749:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941536.244761:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941536.244764:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941536.244765:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941536.244769:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941536.244773:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941536.244777:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941536.244780:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941536.244783:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941536.244786:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324605300 00000400:00000200:0.0:1773941536.244789:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941536.244792:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xd21 [1] + 224 00000800:00000001:0.0:1773941536.244797:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941536.244803:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941536.244805:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941536.244807:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941536.244810:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941536.244814:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10836300 x1860112291484416/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941552 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941536.244821:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d10836300 x1860112291484416/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941552 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941536.244845:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941536.244851:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941536.244853:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941536.244889:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941536.244896:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941536.244898:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941536.244901:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0014b420. 00000400:00000200:2.0:1773941536.244906:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833440 00000400:00000010:2.0:1773941536.244908:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833440. 00000100:00000001:2.0:1773941536.244911:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941536.244917:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10836300 x1860112291484416/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941552 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941536.244943:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10836300 x1860112291484416/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941552 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941536.244950:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941536.244952:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941536.244955:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941536.244958:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941536.244987:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941536.244990:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941536.244993:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941536.244999:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=37 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941536.245003:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=0 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941536.245007:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:2.0:1773941536.245011:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:2.0:1773941536.251802:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941536.251810:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:2.0:1773941536.251816:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1773941536.251821:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1773941536.251827:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1773941536.251831:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941536.251834:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941536.251836:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941536.251838:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:2.0:1773941536.251843:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d10836300 x1860112291484416/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941552 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941536.251853:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941536.251855:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941536.251857:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941536.251860:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941536.251863:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:2.0:1773941536.251865:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941536.251867:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941536.251879:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941536.251880:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941536.251883:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941536.251885:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941536.251887:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1773941536.251890:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1773941536.251893:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941536.251896:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 259 00000400:00000200:2.0:1773941536.251903:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941536.251908:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:2.0:1773941536.251913:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941536.251916:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941536.251919:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941536.251921:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:2.0:1773941536.251925:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941536.251927:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941536.251930:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941536.251932:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941536.251935:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941536.251938:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:2.0:1773941536.251941:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941536.251943:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941536.251946:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941536.251947:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:2.0:1773941536.251951:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:2.0:1773941536.251954:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:2.0:1773941536.251958:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941536.251982:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941536.251987:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941536.251991:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941536.251994:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941536.251996:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941536.252007:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06505880. 00000020:00000040:2.0:1773941536.252012:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:2.0:1773941536.252017:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941536.252019:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941536.252022:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941536.252026:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941536.252030:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941536.252037:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d035a7800. 00000100:00000001:2.0:1773941536.252042:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941536.252049:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d06505880 x1860112291484544/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941536.252059:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941536.252063:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941536.252068:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e62ba1de] to pc [ptlrpcd_rcv+-1] req@ffff988d06505880 x1860112291484544/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941536.252089:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941536.252092:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:2.0:1773941536.252094:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941536.252097:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941536.252100:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941536.252102:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941536.252106:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb66c0 going to sleep for 1 seconds 00000100:00000001:2.0:1773941536.252109:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941536.252112:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941536.252117:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10836300 x1860112291484416/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941552 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941536.252126:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941536.252130:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941536.252132:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941536.252134:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941536.252141:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1773941536.252142:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10836300 x1860112291484416/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941552 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941536.252152:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941536.252158:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941536.252173:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06505880 x1860112291484544/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941536.252175:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10836300 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291484416:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941536.252197:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941536.252203:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941536.252205:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941536.252207:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941536.252218:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 02000000:00000001:3.0:1773941536.252219:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1773941536.252223:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1773941536.252224:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941536.252230:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941536.252239:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06505880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291484544:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000040:2.0:1773941536.252239:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10836300 x1860112291484416/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941552 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941536.252256:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941536.252264:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941536.252267:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941536.252267:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941536.252279:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:2.0:1773941536.252280:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb66c0. 00000100:00000001:2.0:1773941536.252289:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:2.0:1773941536.252291:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:2.0:1773941536.252294:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 02000000:00000010:3.0:1773941536.252299:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03520c00. 00010000:00000001:2.0:1773941536.252300:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 02000000:00000001:3.0:1773941536.252304:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941536.252306:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941536.252312:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00031e00 refcount=4 00000400:00000200:3.0:1773941536.252316:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000020:00000001:2.0:1773941536.252321:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090855936 : -113743618695680 : ffff988d00031e00) 00000400:00000010:3.0:1773941536.252323:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00010000:00000001:2.0:1773941536.252332:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090855936 : -113743618695680 : ffff988d00031e00) 00000400:00000010:3.0:1773941536.252334:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00010000:00010000:2.0:1773941536.252340:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2983 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:3.0:1773941536.252344:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291484544, portal 25 00000100:00000001:3.0:1773941536.252354:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00000001:2.0:1773941536.252358:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941536.252359:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196580480 : -113743512971136 : ffff988d06505880) 00010000:00010000:2.0:1773941536.252370:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2983 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:3.0:1773941536.252375:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06505880 x1860112291484544/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941546 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1773941536.252408:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:2.0:1773941536.252416:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2983 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941536.252417:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941536.252427:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:3.0:1773941536.252434:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182accc0. 00000100:00000200:3.0:1773941536.252443:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291484544, offset 0 00010000:00000001:2.0:1773941536.252444:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:3.0:1773941536.252452:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00000001:2.0:1773941536.252458:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:3.0:1773941536.252464:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:2.0:1773941536.252466:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2983 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941536.252478:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941536.252489:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:2.0:1773941536.252490:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:3.0:1773941536.252495:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00000001:2.0:1773941536.252496:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941536.252502:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941536.252518:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941536.252522:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[62, 63] 00010000:00000001:2.0:1773941536.252522:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:3.0:1773941536.252531:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00010000:00010000:2.0:1773941536.252531:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2983 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941536.252538:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [174:174:256:4294967295] 192.168.204.114@tcp LPNI seq info [64:64:8:4294967295] 00000400:00000200:3.0:1773941536.252559:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:2.0:1773941536.252569:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d00031e00 already unused, calling callback (ffffffffc12b4c20) 10000000:00000001:2.0:1773941536.252577:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:3.0:1773941536.252579:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 10000000:00010000:2.0:1773941536.252583:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2983 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941536.252595:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00010000:00000001:2.0:1773941536.252605:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000800:00000010:3.0:1773941536.252606:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74c00. 00010000:00000001:2.0:1773941536.252610:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:2.0:1773941536.252614:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941536.252616:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00031e00 refcount=5 00000800:00000200:3.0:1773941536.252624:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000020:00000001:2.0:1773941536.252624:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090855936 : -113743618695680 : ffff988d00031e00) 00010000:00000001:2.0:1773941536.252635:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090855936 : -113743618695680 : ffff988d00031e00) 00010000:00000001:2.0:1773941536.252640:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000800:00000200:3.0:1773941536.252645:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00010000:00010000:2.0:1773941536.252648:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2983 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941536.252652:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74c00 type 1, nob 616 niov 1 nkiov 1 10000000:00000001:2.0:1773941536.252660:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:2.0:1773941536.252665:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2983 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1773941536.252673:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000100:00000001:3.0:1773941536.252674:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 10000000:00000040:2.0:1773941536.252683:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000400:00000200:3.0:1773941536.252684:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:3.0:1773941536.252692:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294927848 10000000:00000001:2.0:1773941536.252693:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941536.252707:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:2.0:1773941536.252711:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000100:00000001:3.0:1773941536.252713:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294927839 00000100:00000001:3.0:1773941536.252720:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294927848 00010000:00000001:2.0:1773941536.252725:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000100:00000001:3.0:1773941536.252726:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294927839 00010000:00000001:2.0:1773941536.252731:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000100:00000001:3.0:1773941536.252733:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941536.252737:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941536.252739:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:3.0:1773941536.252741:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941536.252746:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1773941536.252747:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941536.252754:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000020:00000040:2.0:1773941536.252754:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d00031e00 with handle 0x8e86767622e2983 from hash 00000100:00000001:3.0:1773941536.252758:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00010000:00000001:2.0:1773941536.252760:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000100:00000001:3.0:1773941536.252762:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941536.252764:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00010000:00000001:2.0:1773941536.252766:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941536.252767:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941536.252771:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941536.252772:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941536.252777:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1773941536.252781:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:2.0:1773941536.252784:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000800:00000200:1.0:1773941536.252786:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00000001:2.0:1773941536.252788:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941536.252792:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000010:1.0:1773941536.252792:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74c00. 00010000:00000001:2.0:1773941536.252797:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941536.252799:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 10000000:00000001:2.0:1773941536.252801:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941536.252807:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2983 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:1.0:1773941536.252807:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941536.252814:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182accc0 00000400:00000010:1.0:1773941536.252817:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182accc0. 00010000:00000001:2.0:1773941536.252820:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:1.0:1773941536.252821:0:2393:0:(events.c:34:request_out_callback()) Process entered 00010000:00000001:2.0:1773941536.252825:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941536.252829:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000100:00000200:1.0:1773941536.252829:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06505880 x1860112291484544/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941546 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1773941536.252833:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:2.0:1773941536.252837:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941536.252840:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:1.0:1773941536.252842:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00010000:00000001:2.0:1773941536.252843:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000040:1.0:1773941536.252846:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06505880 x1860112291484544/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941546 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:2.0:1773941536.252849:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2983 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941536.252861:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941536.252865:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000040:2.0:1773941536.252870:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d21ec1800 count: 0 00010000:00000001:2.0:1773941536.252891:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941536.252893:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941536.252896:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941536.252903:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10836300 x1860112291484416/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941552 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941536.252911:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941536.252913:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941536.252922:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d02ce5800. 02000000:00000001:2.0:1773941536.252927:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941536.252929:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941536.252931:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941536.252933:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941536.252936:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065c5c00. 02000000:00000001:2.0:1773941536.252940:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941536.252942:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941536.252944:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10836300. 00000100:00000001:2.0:1773941536.252947:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941536.252949:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:2.0:1773941536.252951:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:2.0:1773941536.252954:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:2.0:1773941536.252957:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:2.0:1773941536.253000:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d05437ac0. 00000020:00000010:2.0:1773941536.253006:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d08a4fe00. 00000020:00000010:2.0:1773941536.253012:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d08a4f400. 00000040:00000010:2.0:1773941536.253018:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d03356200. 00000020:00000010:2.0:1773941536.253024:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941536.253030:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d0381f200. 80000000:00000010:2.0:1773941536.253035:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d08bbb900. 00040000:00000010:2.0:1773941536.253040:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d0381e800. 20000000:00000010:2.0:1773941536.253045:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d0381e000. 10000000:00000040:2.0:1773941536.253050:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:2.0:1773941536.253052:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:2.0:1773941536.253056:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d08a4f400. 00000040:00000010:2.0:1773941536.253062:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d03356200. 00000020:00000010:2.0:1773941536.253065:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941536.253070:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d0381f200. 80000000:00000010:2.0:1773941536.253073:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d08bbb900. 00040000:00000010:2.0:1773941536.253076:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d0381e800. 20000000:00000010:2.0:1773941536.253079:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d0381e000. 00000020:00000010:2.0:1773941536.253082:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d08a4fe00. 10000000:00000010:2.0:1773941536.253085:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d05437ac0. 10000000:01000000:2.0:1773941536.253088:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:2.0:1773941536.253097:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:2.0:1773941536.253100:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:2.0:1773941536.253102:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:2.0:1773941536.253104:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:2.0:1773941536.253106:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941536.253108:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941536.254088:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941536.254105:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941536.254114:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941536.254116:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941536.254121:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941536.254127:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941536.254132:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941536.254135:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941536.254138:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941536.254141:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324605380 00000400:00000200:0.0:1773941536.254144:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941536.254148:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xd31 [1] + 0 00000800:00000001:0.0:1773941536.254153:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941536.254159:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941536.254161:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941536.254163:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941536.254166:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941536.254170:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06505880 x1860112291484544/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941546 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941536.254178:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d06505880 x1860112291484544/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941546 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941536.254192:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941536.254196:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941536.254198:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941536.254220:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941536.254223:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941536.254227:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941536.254228:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941536.254231:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941536.254235:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941536.254237:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941536.254240:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941536.254245:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06505880 x1860112291484544/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941546 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941536.254253:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06505880 x1860112291484544/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941546 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941536.254259:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941536.254260:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941536.254262:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941536.254264:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941536.254266:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941536.254268:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941536.254276:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941536.254282:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=37 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941536.254285:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=0 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941536.254287:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941536.254288:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941536.254290:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941536.254292:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941536.254294:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941536.254298:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06505880 x1860112291484544/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941546 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941536.254303:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941536.254304:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941536.254306:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941536.254308:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941536.254315:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941536.254317:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941536.254319:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941536.254321:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941536.254323:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941536.254325:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941536.254327:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941536.254330:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941536 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941536.254332:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941536.254339:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941536.254341:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941536.254345:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941536.254347:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941536.254349:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941536.254353:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941536.254355:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941536.254358:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941536.254363:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941536.254366:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941536.254375:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941536.254595:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941536.254603:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941536.254612:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06505880 x1860112291484544/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941546 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941536.254616:0:14747:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:2.0:1773941536.254628:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06505880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291484544:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00080000:3.0:1773941536.254631:0:14747:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:3.0:1773941536.254640:0:14747:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941536.254643:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941536.254644:0:14747:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941536.254647:0:14747:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000001:2.0:1773941536.254651:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000001:00000001:3.0:1773941536.254653:0:14747:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941536.254657:0:14747:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941536.254664:0:14747:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941536.254665:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06505880 x1860112291484544/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941546 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000001:00000001:3.0:1773941536.254673:0:14747:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941536.254676:0:14747:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941536.254681:0:14747:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941536.254687:0:14747:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941536.254696:0:14747:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941536.254698:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00080000:3.0:1773941536.254701:0:14747:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:3.0:1773941536.254704:0:14747:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:3.0:1773941536.254706:0:14747:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941536.254710:0:14747:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941536.254710:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00080000:3.0:1773941536.254717:0:14747:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 02000000:00000010:2.0:1773941536.254718:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03520c00. 00000100:00000001:3.0:1773941536.254722:0:14747:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941536.254726:0:14747:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:3.0:1773941536.254729:0:14747:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 02000000:00000001:2.0:1773941536.254729:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941536.254742:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941536.254746:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941536.254749:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941536.254751:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d035a7800. 02000000:00000001:2.0:1773941536.254754:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941536.254756:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:3.0:1773941536.254757:0:14747:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:00000010:2.0:1773941536.254758:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06505880. 10000000:00000001:3.0:1773941536.254760:0:14747:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941536.254765:0:14747:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941536.254768:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:02000000:3.0:1773941536.254772:0:14747:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:2.0:1773941536.254773:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941536.254782:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941536.254784:0:14747:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941536.254788:0:14747:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:2.0:1773941536.254790:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000020:00000040:3.0:1773941536.254791:0:14747:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941536.254797:0:14747:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941536.254799:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941536.254802:0:14747:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941536.254806:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941536.254810:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:2.1:1773941536.259005:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d21ec1800. 00010000:00000010:2.1:1773941536.259014:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d00031e00. 00000020:00000001:2.0:1773941536.763171:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941536.763202:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941536.763206:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:0.0:1773941537.403073:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1773941537.403081:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941537.403085:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941537.403088:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941537.403090:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941537.403091:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1773941537.403094:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:0.0:1773941537.403097:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:0.0:1773941537.403100:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:0.0:1773941537.403103:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1773941537.403105:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:0.0:1773941537.403116:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10837480. 00000020:00000040:0.0:1773941537.403138:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941537.403144:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941537.403147:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941537.403153:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941537.403160:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065c4a00. 00000100:00000001:0.0:1773941537.403167:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941537.403173:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1773941537.403176:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1773941537.403184:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d21ec1400. 00010000:00000001:0.0:1773941537.403191:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1773941537.403194:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d00030a00. 00000020:00000001:0.0:1773941537.403199:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:0.0:1773941537.403201:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d00030a00 with handle 0x8e86767622e298a to hash 00000020:00000001:0.0:1773941537.403204:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:0.0:1773941537.403206:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330090850816 : -113743618700800 : ffff988d00030a00) 00010000:00000001:0.0:1773941537.403208:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330090850816 : -113743618700800 : ffff988d00030a00) 00010000:00000001:0.0:1773941537.403211:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941537.403213:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941537.403217:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00030a00/0x8e86767622e298a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941537.403228:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d00030a00/0x8e86767622e298a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941537.403240:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d00030a00/0x8e86767622e298a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941537.403247:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1773941537.403249:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941537.403256:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb6a80. 00000100:00000001:0.0:1773941537.403258:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330139634304 : -113743569917312 : ffff988d02eb6a80) 00000100:00000001:0.0:1773941537.403261:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941537.403263:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367702144 : -113743341849472 : ffff988d10837480) 00000100:00000001:0.0:1773941537.403268:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941537.403277:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941537.403280:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941537.403282:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941537.403293:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10837480 x1860112291484672/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941537.403304:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941537.403306:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941537.403310:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941537.403313:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941537.403316:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941537.403320:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10837480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291484672:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941537.403325:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941537.403330:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941537.403332:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941537.403335:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941537.403340:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d02ce6800. 02000000:00000001:0.0:1773941537.403342:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941537.403348:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941537.403351:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0014bbb0. 00000400:00000010:0.0:1773941537.403356:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833550. 00000100:00000200:0.0:1773941537.403362:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291484672, portal 25 00000100:00000001:0.0:1773941537.403363:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941537.403364:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367702144 : -113743341849472 : ffff988d10837480) 00000100:00000040:0.0:1773941537.403368:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10837480 x1860112291484672/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941553 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941537.403374:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941537.403376:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941537.403377:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833660. 00000100:00000200:0.0:1773941537.403380:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291484672, offset 0 00000400:00000200:0.0:1773941537.403385:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941537.403413:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941537.403421:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941537.403425:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941537.403433:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941537.403437:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941537.403439:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941537.403441:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[64, 63] 00000400:00000200:0.0:1773941537.403444:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:0.0:1773941537.403447:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [175:175:256:4294967295] 192.168.204.124@tcp LPNI seq info [65:65:8:4294967295] 00000400:00000200:0.0:1773941537.403450:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941537.403458:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:0.0:1773941537.403463:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:0.0:1773941537.403467:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a20bd00. 00000800:00000200:0.0:1773941537.403472:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:0.0:1773941537.403477:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:0.0:1773941537.403481:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a20bd00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1773941537.403513:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941537.403518:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941537.403531:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294928999 00000100:00000001:0.0:1773941537.403539:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294927848 00000100:00000001:0.0:1773941537.403544:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294928999 00000100:00000001:0.0:1773941537.403547:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294927848 00000100:00000001:0.0:1773941537.403550:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941537.403553:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941537.403555:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941537.403557:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1773941537.403561:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6a80 going to sleep for 11 seconds 00000100:00000001:0.0:1773941537.403564:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941537.403567:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941537.403693:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941537.403699:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a20bd00. 00000400:00000200:1.0:1773941537.403705:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941537.403712:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941537.403719:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833660 00000400:00000010:1.0:1773941537.403721:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833660. 00000100:00000001:1.0:1773941537.403725:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941537.403731:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10837480 x1860112291484672/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941553 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941537.403742:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941537.403753:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d10837480 x1860112291484672/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941553 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941537.403762:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941537.403764:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941537.404856:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941537.404868:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941537.404870:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941537.404872:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941537.404876:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941537.404886:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941537.404891:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941537.404894:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941537.404899:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941537.404902:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324605400 00000400:00000200:0.0:1773941537.404905:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941537.404910:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xd39 [1] + 224 00000800:00000001:0.0:1773941537.404917:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941537.404926:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941537.404930:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941537.404936:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941537.404941:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941537.404946:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10837480 x1860112291484672/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941553 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941537.404958:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d10837480 x1860112291484672/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941553 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941537.405001:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941537.405009:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941537.405013:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941537.405050:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941537.405062:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941537.405064:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941537.405067:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0014bbb0. 00000400:00000200:2.0:1773941537.405073:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833550 00000400:00000010:2.0:1773941537.405075:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833550. 00000100:00000001:2.0:1773941537.405079:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941537.405088:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10837480 x1860112291484672/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941553 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941537.405100:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10837480 x1860112291484672/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941553 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941537.405107:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941537.405108:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941537.405111:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941537.405114:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941537.405116:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941537.405118:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941537.405121:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941537.405128:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=38 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941537.405131:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941537.405133:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:2.0:1773941537.405137:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:2.0:1773941537.405141:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941537.405144:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:2.0:1773941537.405146:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1773941537.405150:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1773941537.405154:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1773941537.405157:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941537.405159:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941537.405161:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941537.405178:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:2.0:1773941537.405182:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d10837480 x1860112291484672/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941553 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941537.405189:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941537.405191:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941537.405193:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941537.405196:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941537.405199:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:2.0:1773941537.405200:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941537.405202:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941537.405203:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941537.405204:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941537.405205:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941537.405207:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941537.405210:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1773941537.405211:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1773941537.405214:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941537.405217:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 259 00000400:00000200:2.0:1773941537.405223:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941537.405228:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:2.0:1773941537.405231:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941537.405234:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941537.405236:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941537.405238:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:2.0:1773941537.405241:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941537.405243:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941537.405246:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941537.405249:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941537.405251:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941537.405252:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:2.0:1773941537.405255:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941537.405256:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941537.405258:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941537.405259:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:2.0:1773941537.405266:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:2.0:1773941537.405270:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:2.0:1773941537.405272:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941537.405274:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941537.405278:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941537.405280:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941537.405283:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941537.405284:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941537.405289:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06505880. 00000020:00000040:2.0:1773941537.405292:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:2.0:1773941537.405296:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941537.405297:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941537.405300:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941537.405302:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941537.405305:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941537.405311:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d035a4400. 00000100:00000001:2.0:1773941537.405316:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941537.405321:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d06505880 x1860112291484800/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941537.405329:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941537.405332:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941537.405336:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e62ba1de] to pc [ptlrpcd_rcv+-1] req@ffff988d06505880 x1860112291484800/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941537.405357:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941537.405360:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:2.0:1773941537.405361:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941537.405363:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941537.405366:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941537.405367:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941537.405369:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6a80 going to sleep for 1 seconds 00000100:00000001:2.0:1773941537.405371:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941537.405373:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941537.405377:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10837480 x1860112291484672/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941553 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941537.405388:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941537.405392:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941537.405394:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941537.405396:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941537.405401:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10837480 x1860112291484672/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941553 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941537.405408:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10837480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291484672:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941537.405411:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941537.405413:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941537.405418:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941537.405419:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941537.405421:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:3.0:1773941537.405424:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1773941537.405425:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941537.405432:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10837480 x1860112291484672/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941553 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941537.405441:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06505880 x1860112291484800/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941537.405442:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941537.405447:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb6a80. 00000100:00000001:2.0:1773941537.405452:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:2.0:1773941537.405455:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:2.0:1773941537.405461:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:2.0:1773941537.405464:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:2.0:1773941537.405465:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941537.405467:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030a00 refcount=4 00000020:00000001:2.0:1773941537.405469:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090850816 : -113743618700800 : ffff988d00030a00) 00010000:00000001:2.0:1773941537.405474:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090850816 : -113743618700800 : ffff988d00030a00) 00010000:00010000:2.0:1773941537.405504:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d00030a00/0x8e86767622e298a lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941537.405514:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941537.405518:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941537.405522:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 02000000:00000001:3.0:1773941537.405526:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00010000:00010000:2.0:1773941537.405531:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d00030a00/0x8e86767622e298a lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941537.405532:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941537.405537:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941537.405546:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06505880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291484800:192.168.204.114@tcp:250:ll_cfg_requeue.0 00010000:00000001:2.0:1773941537.405557:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000001:3.0:1773941537.405558:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941537.405566:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00010000:2.0:1773941537.405566:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00030a00/0x8e86767622e298a lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941537.405571:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941537.405577:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:2.0:1773941537.405585:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 02000000:00000010:3.0:1773941537.405586:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03521c00. 00010000:00000001:2.0:1773941537.405590:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 02000000:00000001:3.0:1773941537.405592:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941537.405596:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d00030a00/0x8e86767622e298a lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941537.405605:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941537.405614:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00010000:00000001:2.0:1773941537.405615:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941537.405623:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1773941537.405625:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00010000:00000001:2.0:1773941537.405628:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:2.0:1773941537.405634:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d00030a00/0x8e86767622e298a lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:3.0:1773941537.405635:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291484800, portal 25 00000100:00000001:3.0:1773941537.405643:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941537.405647:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196580480 : -113743512971136 : ffff988d06505880) 00010000:00010000:2.0:1773941537.405651:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d00030a00 already unused, calling callback (ffffffffc12b4c20) 10000000:00000001:2.0:1773941537.405658:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000040:3.0:1773941537.405660:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06505880 x1860112291484800/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941547 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00010000:2.0:1773941537.405663:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d00030a00/0x8e86767622e298a lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941537.405687:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000100:00000001:3.0:1773941537.405692:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:2.0:1773941537.405692:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:2.0:1773941537.405696:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941537.405698:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030a00 refcount=5 00000100:00000040:3.0:1773941537.405702:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000020:00000001:2.0:1773941537.405703:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090850816 : -113743618700800 : ffff988d00030a00) 00010000:00000001:2.0:1773941537.405709:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090850816 : -113743618700800 : ffff988d00030a00) 00000400:00000010:3.0:1773941537.405711:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac770. 00010000:00000001:2.0:1773941537.405714:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000100:00000200:3.0:1773941537.405720:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291484800, offset 0 00010000:00010000:2.0:1773941537.405721:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d00030a00/0x8e86767622e298a lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941537.405736:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 10000000:00000001:2.0:1773941537.405738:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:2.0:1773941537.405751:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d00030a00/0x8e86767622e298a lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941537.405756:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 10000000:01000000:2.0:1773941537.405768:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:3.0:1773941537.405770:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 10000000:00000040:2.0:1773941537.405777:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000400:00000200:3.0:1773941537.405782:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 10000000:00000001:2.0:1773941537.405783:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941537.405790:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000400:00000200:3.0:1773941537.405791:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00000001:2.0:1773941537.405794:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:2.0:1773941537.405799:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000400:00000200:3.0:1773941537.405800:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:2.0:1773941537.405803:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1773941537.405808:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:3.0:1773941537.405810:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:2.0:1773941537.405812:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941537.405816:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d00030a00 with handle 0x8e86767622e298a from hash 00010000:00000001:2.0:1773941537.405819:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1773941537.405822:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941537.405825:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941537.405829:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000400:00000200:3.0:1773941537.405831:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[64, 65] 00010000:00000001:2.0:1773941537.405832:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:2.0:1773941537.405837:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000400:00000200:3.0:1773941537.405844:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00010000:00000001:2.0:1773941537.405844:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941537.405848:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941537.405851:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941537.405856:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [176:176:256:4294967295] 192.168.204.114@tcp LPNI seq info [66:66:8:4294967295] 10000000:00000001:2.0:1773941537.405857:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941537.405864:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d00030a00/0x8e86767622e298a lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941537.405873:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:2.0:1773941537.405875:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941537.405879:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941537.405880:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:2.0:1773941537.405882:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:2.0:1773941537.405884:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941537.405887:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941537.405889:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1773941537.405893:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d00030a00/0x8e86767622e298a lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941537.405899:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00010000:00000040:2.0:1773941537.405904:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d21ec1400 count: 0 00000800:00000200:3.0:1773941537.405911:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:3.0:1773941537.405920:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74f00. 00010000:00000001:2.0:1773941537.405925:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941537.405929:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000800:00000200:3.0:1773941537.405931:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000100:00000001:2.0:1773941537.405934:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941537.405942:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10837480 x1860112291484672/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941553 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1773941537.405950:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:3.0:1773941537.405958:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74f00 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:2.0:1773941537.406031:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1773941537.406033:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941537.406037:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000400:00000200:3.0:1773941537.406044:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 02000000:00000010:2.0:1773941537.406045:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d02ce6800. 00000100:00000001:3.0:1773941537.406053:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294929002 02000000:00000001:2.0:1773941537.406058:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941537.406063:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941537.406067:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941537.406071:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294928999 00000020:00000001:2.0:1773941537.406072:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941537.406078:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065c4a00. 00000100:00000001:3.0:1773941537.406083:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294929002 02000000:00000001:2.0:1773941537.406086:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941537.406090:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1773941537.406093:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294928999 00000100:00000010:2.0:1773941537.406094:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10837480. 00000100:00000001:2.0:1773941537.406103:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941537.406105:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941537.406106:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:2.0:1773941537.406114:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941537.406116:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941537.406116:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 10000000:01000000:2.0:1773941537.406120:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000800:00000010:1.0:1773941537.406129:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74f00. 00000100:00000001:3.0:1773941537.406131:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941537.406131:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000400:00000200:1.0:1773941537.406137:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1773941537.406140:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:00000010:2.0:1773941537.406141:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d05437ac0. 00000400:00000200:1.0:1773941537.406147:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941537.406153:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac770 00000020:00000010:2.0:1773941537.406154:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d0381f200. 00000100:00000001:3.0:1773941537.406156:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000400:00000010:1.0:1773941537.406164:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac770. 00000020:00000010:2.0:1773941537.406169:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d0381e000. 00000100:00000001:1.0:1773941537.406169:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000040:00000010:2.0:1773941537.406179:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d0381e800. 00000100:00000200:1.0:1773941537.406180:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06505880 x1860112291484800/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941547 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1773941537.406192:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d09305000. 00000100:00000001:1.0:1773941537.406192:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941537.406202:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06505880 x1860112291484800/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941547 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1773941537.406203:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d03356200. 80000000:00000010:2.0:1773941537.406212:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d08bbb900. 00000100:00000001:1.0:1773941537.406215:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941537.406218:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00040000:00000010:2.0:1773941537.406237:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d08a4fe00. 20000000:00000010:2.0:1773941537.406245:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d08a4f400. 00000100:00000001:3.0:1773941537.406247:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 10000000:00000040:2.0:1773941537.406251:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:2.0:1773941537.406258:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941537.406259:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941537.406266:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000020:00000010:2.0:1773941537.406266:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d0381e000. 00000100:00000001:3.0:1773941537.406275:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000040:00000010:2.0:1773941537.406278:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d0381e800. 00000100:00000001:3.0:1773941537.406283:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1773941537.406294:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941537.406300:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d03356200. 80000000:00000010:2.0:1773941537.406305:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d08bbb900. 00040000:00000010:2.0:1773941537.406311:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d08a4fe00. 20000000:00000010:2.0:1773941537.406317:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d08a4f400. 00000020:00000010:2.0:1773941537.406320:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0381f200. 10000000:00000010:2.0:1773941537.406324:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d05437ac0. 10000000:01000000:2.0:1773941537.406327:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:2.0:1773941537.406331:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:2.0:1773941537.406334:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 00000800:00000001:0.0:1773941537.407645:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941537.407653:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941537.407656:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941537.407657:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941537.407661:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941537.407665:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941537.407669:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941537.407672:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941537.407674:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941537.407677:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324605480 00000400:00000200:0.0:1773941537.407679:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941537.407683:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xd49 [1] + 0 00000800:00000001:0.0:1773941537.407687:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941537.407699:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941537.407700:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941537.407703:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941537.407706:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941537.407710:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06505880 x1860112291484800/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941547 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941537.407719:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d06505880 x1860112291484800/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941547 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941537.407733:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941537.407737:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941537.407739:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941537.407812:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941537.407816:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941537.407821:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941537.407823:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941537.407827:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941537.407833:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941537.407844:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941537.407849:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941537.407857:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06505880 x1860112291484800/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941547 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941537.407872:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06505880 x1860112291484800/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941547 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941537.407881:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941537.407883:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941537.407886:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941537.407889:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941537.407891:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941537.407893:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941537.407896:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941537.407900:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=38 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941537.407905:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941537.407909:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941537.407911:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941537.407915:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941537.407917:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941537.407921:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941537.407927:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06505880 x1860112291484800/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941547 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941537.407937:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941537.407941:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941537.407943:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941537.407946:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941537.407953:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941537.407957:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941537.407990:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941537.407994:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941537.407997:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941537.408000:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941537.408005:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941537.408011:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941537 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941537.408022:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941537.408031:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941537.408034:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941537.408036:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941537.408039:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941537.408041:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941537.408046:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941537.408049:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941537.408052:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941537.408057:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941537.408061:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941537.408065:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941537.408393:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941537.408400:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941537.408413:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06505880 x1860112291484800/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941547 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941537.408419:0:14748:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00080000:1.0:1773941537.408441:0:14748:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:1.0:1773941537.408447:0:14748:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00100000:3.0:1773941537.408458:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06505880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291484800:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941537.408475:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941537.408478:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941537.408500:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06505880 x1860112291484800/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941547 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941537.408500:0:14748:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941537.408504:0:14748:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:1.0:1773941537.408511:0:14748:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941537.408516:0:14748:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941537.408520:0:14748:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941537.408525:0:14748:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:3.0:1773941537.408528:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:1.0:1773941537.408529:0:14748:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:3.0:1773941537.408540:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:1.0:1773941537.408540:0:14748:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941537.408547:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03521c00. 10000000:00000001:1.0:1773941537.408548:0:14748:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941537.408555:0:14748:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000001:3.0:1773941537.408556:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941537.408561:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000100:00080000:1.0:1773941537.408562:0:14748:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000020:00000040:3.0:1773941537.408564:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941537.408570:0:14748:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000020:00000001:3.0:1773941537.408571:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941537.408575:0:14748:0:(recover.c:147:ptlrpc_resend()) Process entered 02000000:00000010:3.0:1773941537.408578:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d035a4400. 02000000:00000001:3.0:1773941537.408582:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941537.408587:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1773941537.408590:0:14748:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941537.408591:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06505880. 00000100:00080000:1.0:1773941537.408596:0:14748:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941537.408600:0:14748:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941537.408601:0:14748:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941537.408606:0:14748:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00000001:3.0:1773941537.408611:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941537.408615:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941537.408619:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941537.408624:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:1.0:1773941537.408625:0:14748:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:00000001:3.0:1773941537.408627:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941537.408629:0:14748:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941537.408635:0:14748:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941537.408636:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941537.408642:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:02000000:1.0:1773941537.408643:0:14748:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:1.0:1773941537.408655:0:14748:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941537.408658:0:14748:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941537.408660:0:14748:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941537.408662:0:14748:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941537.408665:0:14748:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941537.412645:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:2.0:1773941537.412650:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:2.0:1773941537.412652:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941537.412657:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00010000:00000010:2.1:1773941537.413999:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d21ec1400. 00010000:00000010:2.1:1773941537.414005:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d00030a00. 00000020:00000001:3.0:1773941537.787075:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941537.787083:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941537.787088:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941538.683067:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294929002, at 4294929024 00000100:00000040:0.0:1773941538.683067:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294929002, at 4294929024 00000100:00000010:1.0:1773941538.683081:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d08bbbc00. 00000100:00000010:0.0:1773941538.683084:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d08bbb120. 00000020:00000001:2.0:1773941538.811082:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941538.811088:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941538.811092:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:2.0:1773941539.003066:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:2.0:1773941539.003072:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941539.003076:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941539.003078:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941539.003079:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941539.003080:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:2.0:1773941539.003082:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:2.0:1773941539.003084:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:2.0:1773941539.003087:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:2.0:1773941539.003089:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:2.0:1773941539.003090:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:2.0:1773941539.003100:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06506a00. 00000020:00000040:2.0:1773941539.003104:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941539.003108:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941539.003111:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941539.003115:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941539.003122:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d08a4f400. 00000100:00000001:2.0:1773941539.003127:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941539.003131:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:2.0:1773941539.003148:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:2.0:1773941539.003155:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d071eec00. 00010000:00000001:2.0:1773941539.003160:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:2.0:1773941539.003162:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d089de800. 00000020:00000001:2.0:1773941539.003166:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:2.0:1773941539.003168:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d089de800 with handle 0x8e86767622e2991 to hash 00000020:00000001:2.0:1773941539.003169:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:2.0:1773941539.003170:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330235217920 : -113743474333696 : ffff988d089de800) 00010000:00000001:2.0:1773941539.003173:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330235217920 : -113743474333696 : ffff988d089de800) 00010000:00000001:2.0:1773941539.003177:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941539.003180:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941539.003183:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2991 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941539.003192:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2991 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941539.003201:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2991 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941539.003206:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:2.0:1773941539.003208:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:2.0:1773941539.003212:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d09350540. 00000100:00000001:2.0:1773941539.003214:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330245121344 : -113743464430272 : ffff988d09350540) 00000100:00000001:2.0:1773941539.003216:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941539.003217:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584960 : -113743512966656 : ffff988d06506a00) 00000100:00000001:2.0:1773941539.003221:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941539.003225:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941539.003226:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:2.0:1773941539.003228:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941539.003236:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06506a00 x1860112291484928/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941539.003243:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941539.003248:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941539.003251:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941539.003253:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941539.003255:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941539.003257:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291484928:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941539.003260:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941539.003264:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941539.003266:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941539.003268:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941539.003271:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a4000. 02000000:00000001:2.0:1773941539.003273:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941539.003279:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941539.003283:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000400:00000010:2.0:1773941539.003288:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000200:2.0:1773941539.003293:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291484928, portal 25 00000100:00000001:2.0:1773941539.003295:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941539.003296:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584960 : -113743512966656 : ffff988d06506a00) 00000100:00000040:2.0:1773941539.003299:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06506a00 x1860112291484928/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941555 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941539.003305:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941539.003307:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:2.0:1773941539.003308:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516a18. 00000100:00000200:2.0:1773941539.003311:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291484928, offset 0 00000400:00000200:2.0:1773941539.003315:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941539.003322:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941539.003328:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941539.003332:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941539.003336:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941539.003340:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941539.003342:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941539.003344:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[66, 65] 00000400:00000200:2.0:1773941539.003347:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:2.0:1773941539.003350:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [177:177:256:4294967295] 192.168.204.124@tcp LPNI seq info [67:67:8:4294967295] 00000400:00000200:2.0:1773941539.003356:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:2.0:1773941539.003365:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:2.0:1773941539.003369:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:2.0:1773941539.003373:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0dbb00. 00000800:00000200:2.0:1773941539.003378:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:2.0:1773941539.003384:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:2.0:1773941539.003388:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0dbb00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:2.0:1773941539.003403:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941539.003407:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941539.003409:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294930599 00000100:00000010:2.0:1773941539.003412:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d08bbb900. 00000100:00000001:2.0:1773941539.003417:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1773941539.003423:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294930599 00000100:00000010:2.0:1773941539.003425:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d08bbbf00. 00000100:00000001:2.0:1773941539.003427:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1773941539.003429:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941539.003431:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941539.003433:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941539.003435:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:2.0:1773941539.003437:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d09350540 going to sleep for 11 seconds 00000100:00000001:2.0:1773941539.003440:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941539.003442:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941539.003581:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941539.003587:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0dbb00. 00000400:00000200:1.0:1773941539.003595:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941539.003601:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941539.003607:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516a18 00000400:00000010:1.0:1773941539.003610:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516a18. 00000100:00000001:1.0:1773941539.003614:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941539.003619:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06506a00 x1860112291484928/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941555 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941539.003630:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941539.003634:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d06506a00 x1860112291484928/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941555 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941539.003642:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941539.003645:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941539.004701:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941539.004713:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941539.004716:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941539.004719:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941539.004725:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941539.004731:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941539.004738:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941539.004742:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941539.004746:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941539.004750:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324605500 00000400:00000200:0.0:1773941539.004755:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941539.004762:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xd51 [1] + 224 00000800:00000001:0.0:1773941539.004769:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941539.004779:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941539.004781:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941539.004785:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941539.004789:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941539.004795:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06506a00 x1860112291484928/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941555 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941539.004821:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d06506a00 x1860112291484928/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941555 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941539.004840:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941539.004848:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941539.004852:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941539.004868:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941539.004873:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941539.004875:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941539.004877:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941539.004880:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941539.004882:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941539.004884:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941539.004888:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06506a00 x1860112291484928/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941555 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941539.004899:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06506a00 x1860112291484928/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941555 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941539.004915:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941539.004917:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941539.004920:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941539.004922:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941539.004925:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941539.004928:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941539.004931:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941539.004937:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=40 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941539.004941:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941539.004944:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:2.0:1773941539.004948:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:2.0:1773941539.004952:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941539.004955:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:2.0:1773941539.004958:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1773941539.004981:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1773941539.004987:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1773941539.004991:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941539.004993:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941539.004995:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941539.004997:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:2.0:1773941539.005001:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d06506a00 x1860112291484928/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941555 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941539.005009:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941539.005011:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941539.005013:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941539.005015:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941539.005018:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:2.0:1773941539.005020:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941539.005022:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941539.005023:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941539.005024:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941539.005026:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941539.005033:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941539.005036:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1773941539.005038:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1773941539.005040:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941539.005044:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 260 00000400:00000200:2.0:1773941539.005048:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941539.005053:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:2.0:1773941539.005057:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941539.005060:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941539.005063:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941539.005065:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:2.0:1773941539.005069:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941539.005071:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941539.005073:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941539.005075:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941539.005078:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941539.005080:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:2.0:1773941539.005083:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941539.005085:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941539.005087:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941539.005089:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:2.0:1773941539.005091:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:2.0:1773941539.005095:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:2.0:1773941539.005098:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941539.005101:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941539.005104:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941539.005107:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941539.005110:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941539.005112:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941539.005116:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06506680. 00000020:00000040:2.0:1773941539.005119:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:2.0:1773941539.005122:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941539.005129:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941539.005131:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941539.005133:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941539.005134:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941539.005138:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d035a7400. 00000100:00000001:2.0:1773941539.005141:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941539.005146:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d06506680 x1860112291485056/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941539.005151:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941539.005153:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941539.005156:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000022f6a110] to pc [ptlrpcd_rcv+-1] req@ffff988d06506680 x1860112291485056/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941539.005172:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941539.005174:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:2.0:1773941539.005175:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941539.005177:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941539.005179:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941539.005180:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941539.005182:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d09350540 going to sleep for 1 seconds 00000100:00000001:2.0:1773941539.005184:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941539.005186:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941539.005190:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06506a00 x1860112291484928/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941555 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941539.005195:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941539.005197:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941539.005198:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941539.005202:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941539.005203:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941539.005204:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941539.005206:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941539.005211:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06506a00 x1860112291484928/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941555 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941539.005215:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06506680 x1860112291485056/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941539.005223:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291484928:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941539.005231:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941539.005233:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941539.005234:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941539.005240:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1773941539.005240:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:3.0:1773941539.005244:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941539.005245:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:2.0:1773941539.005248:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1773941539.005249:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941539.005253:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06506a00 x1860112291484928/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941555 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941539.005255:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06506680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291485056:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941539.005263:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941539.005266:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941539.005269:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941539.005269:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941539.005274:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:2.0:1773941539.005274:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d09350540. 00000100:00000001:2.0:1773941539.005279:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 02000000:00000010:3.0:1773941539.005280:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03521c00. 00000100:00000001:2.0:1773941539.005281:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:3.0:1773941539.005285:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941539.005286:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:2.0:1773941539.005289:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:3.0:1773941539.005290:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000020:00000001:2.0:1773941539.005292:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000010:3.0:1773941539.005295:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00000020:00000040:2.0:1773941539.005295:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089de800 refcount=4 00000020:00000001:2.0:1773941539.005301:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235217920 : -113743474333696 : ffff988d089de800) 00010000:00000001:2.0:1773941539.005303:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235217920 : -113743474333696 : ffff988d089de800) 00010000:00010000:2.0:1773941539.005306:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2991 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:3.0:1773941539.005310:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00010000:00000001:2.0:1773941539.005312:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000200:3.0:1773941539.005315:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291485056, portal 25 00010000:00010000:2.0:1773941539.005317:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2991 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941539.005320:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941539.005323:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584064 : -113743512967552 : ffff988d06506680) 00010000:00000001:2.0:1773941539.005327:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000040:3.0:1773941539.005331:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06506680 x1860112291485056/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941549 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:2.0:1773941539.005332:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2991 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941539.005349:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941539.005350:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:2.0:1773941539.005352:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2991 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941539.005362:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:3.0:1773941539.005367:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941539.005372:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00000001:2.0:1773941539.005373:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1773941539.005375:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac198. 00010000:00000001:2.0:1773941539.005378:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000100:00000200:3.0:1773941539.005380:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291485056, offset 0 00010000:00010000:2.0:1773941539.005381:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2991 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941539.005388:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00010000:2.0:1773941539.005391:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d089de800 already unused, calling callback (ffffffffc12b4c20) 00000400:00000200:3.0:1773941539.005395:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 10000000:00000001:2.0:1773941539.005396:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:2.0:1773941539.005400:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2991 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941539.005403:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941539.005410:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:2.0:1773941539.005411:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:2.0:1773941539.005414:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:3.0:1773941539.005415:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000020:00000001:2.0:1773941539.005418:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:3.0:1773941539.005421:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000020:00000040:2.0:1773941539.005422:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089de800 refcount=5 00000400:00000200:3.0:1773941539.005427:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000020:00000001:2.0:1773941539.005428:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235217920 : -113743474333696 : ffff988d089de800) 00010000:00000001:2.0:1773941539.005433:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235217920 : -113743474333696 : ffff988d089de800) 00000400:00000200:3.0:1773941539.005434:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[66, 67] 00010000:00000001:2.0:1773941539.005437:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:3.0:1773941539.005441:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00010000:00010000:2.0:1773941539.005442:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2991 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941539.005447:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [178:178:256:4294967295] 192.168.204.114@tcp LPNI seq info [68:68:8:4294967295] 00000400:00000200:3.0:1773941539.005470:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 10000000:00000001:2.0:1773941539.005472:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:2.0:1773941539.005475:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2991 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941539.005481:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 10000000:01000000:2.0:1773941539.005487:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000800:00000200:3.0:1773941539.005493:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 10000000:00000040:2.0:1773941539.005494:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:2.0:1773941539.005497:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1773941539.005500:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74500. 00010000:00010000:2.0:1773941539.005500:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:2.0:1773941539.005503:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:2.0:1773941539.005507:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000800:00000200:3.0:1773941539.005508:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00010000:00000001:2.0:1773941539.005509:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1773941539.005513:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941539.005514:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941539.005519:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000020:00000040:2.0:1773941539.005519:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d089de800 with handle 0x8e86767622e2991 from hash 00000800:00000200:3.0:1773941539.005522:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74500 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:2.0:1773941539.005528:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1773941539.005529:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941539.005530:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941539.005531:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1773941539.005533:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000100:00000001:3.0:1773941539.005534:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941539.005535:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:2.0:1773941539.005539:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:3.0:1773941539.005541:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:2.0:1773941539.005542:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941539.005545:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941539.005546:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294930601 10000000:00000001:2.0:1773941539.005548:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941539.005552:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294930599 00010000:00010000:2.0:1773941539.005552:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2991 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941539.005560:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294930601 00010000:00000001:2.0:1773941539.005560:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941539.005563:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:3.0:1773941539.005566:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294930599 00010000:00000001:2.0:1773941539.005566:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:2.0:1773941539.005568:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:2.0:1773941539.005571:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941539.005573:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:3.0:1773941539.005574:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:2.0:1773941539.005575:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941539.005577:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941539.005579:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2991 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:2.0:1773941539.005585:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d071eec00 count: 0 00000100:00000001:3.0:1773941539.005588:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941539.005592:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:2.0:1773941539.005596:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:3.0:1773941539.005598:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:2.0:1773941539.005599:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941539.005602:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941539.005603:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1773941539.005606:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941539.005609:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000040:2.0:1773941539.005610:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06506a00 x1860112291484928/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941555 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941539.005611:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941539.005616:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1773941539.005617:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1773941539.005624:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74500. 00000100:00000001:2.0:1773941539.005626:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941539.005629:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941539.005633:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a4000. 00000400:00000200:1.0:1773941539.005633:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:2.0:1773941539.005638:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941539.005640:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941539.005643:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000400:00000200:1.0:1773941539.005644:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000020:00000001:2.0:1773941539.005646:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941539.005655:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d08a4f400. 00000400:00000200:1.0:1773941539.005658:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac198 02000000:00000001:2.0:1773941539.005660:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941539.005664:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000400:00000010:1.0:1773941539.005665:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac198. 00000100:00000010:2.0:1773941539.005666:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06506a00. 00000100:00000001:2.0:1773941539.005672:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941539.005673:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1773941539.005674:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:2.0:1773941539.005677:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:2.0:1773941539.005679:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:2.0:1773941539.005682:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000100:00000200:1.0:1773941539.005682:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06506680 x1860112291485056/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941549 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000010:2.0:1773941539.005686:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d05437ac0. 00000020:00000010:2.0:1773941539.005691:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d08a4f400. 00000020:00000010:2.0:1773941539.005697:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d08a4fe00. 00000100:00000001:1.0:1773941539.005699:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000040:00000010:2.0:1773941539.005703:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d03356200. 00000100:00000040:1.0:1773941539.005706:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06506680 x1860112291485056/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941549 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1773941539.005708:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941539.005715:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d0381f200. 00000100:00000001:1.0:1773941539.005718:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 80000000:00000010:2.0:1773941539.005719:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d08bbbde0. 00000100:00000001:1.0:1773941539.005723:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00040000:00000010:2.0:1773941539.005725:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d0381e800. 20000000:00000010:2.0:1773941539.005729:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d0381e000. 10000000:00000040:2.0:1773941539.005733:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:2.0:1773941539.005735:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:2.0:1773941539.005738:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d08a4fe00. 00000040:00000010:2.0:1773941539.005743:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d03356200. 00000020:00000010:2.0:1773941539.005746:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941539.005749:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d0381f200. 80000000:00000010:2.0:1773941539.005751:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d08bbbde0. 00040000:00000010:2.0:1773941539.005753:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d0381e800. 20000000:00000010:2.0:1773941539.005755:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d0381e000. 00000020:00000010:2.0:1773941539.005757:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d08a4f400. 10000000:00000010:2.0:1773941539.005759:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d05437ac0. 10000000:01000000:2.0:1773941539.005761:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:2.0:1773941539.005763:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:2.0:1773941539.005765:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:2.0:1773941539.005766:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:2.0:1773941539.005767:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:2.0:1773941539.005771:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941539.005773:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941539.006985:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941539.006991:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941539.006992:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941539.006995:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941539.006998:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941539.007002:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941539.007006:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941539.007009:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941539.007012:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941539.007014:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324605580 00000400:00000200:0.0:1773941539.007017:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941539.007019:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xd59 [1] + 0 00000800:00000001:0.0:1773941539.007023:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941539.007026:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941539.007028:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941539.007030:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941539.007033:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941539.007037:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06506680 x1860112291485056/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941549 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941539.007045:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d06506680 x1860112291485056/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941549 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941539.007056:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941539.007060:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941539.007063:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941539.007073:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941539.007076:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941539.007079:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941539.007080:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941539.007082:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941539.007086:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941539.007087:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941539.007089:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941539.007094:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06506680 x1860112291485056/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941549 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941539.007102:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06506680 x1860112291485056/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941549 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941539.007107:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941539.007108:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941539.007110:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941539.007111:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941539.007112:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941539.007113:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941539.007115:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941539.007117:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=40 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941539.007120:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941539.007121:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941539.007123:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941539.007124:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941539.007125:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941539.007127:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941539.007130:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06506680 x1860112291485056/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941549 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941539.007145:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941539.007146:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941539.007148:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941539.007149:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941539.007153:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941539.007154:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941539.007155:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941539.007156:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941539.007158:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941539.007159:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941539.007161:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941539.007163:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941539 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941539.007165:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941539.007170:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941539.007171:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941539.007172:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941539.007174:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941539.007175:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941539.007177:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941539.007179:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941539.007181:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941539.007183:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941539.007186:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941539.007188:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941539.007369:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941539.007372:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941539.007378:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06506680 x1860112291485056/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941549 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941539.007385:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06506680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291485056:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941539.007388:0:14749:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941539.007389:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941539.007393:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941539.007396:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06506680 x1860112291485056/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941549 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1773941539.007400:0:14749:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00000001:2.0:1773941539.007404:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00100000:3.0:1773941539.007407:0:14749:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000001:2.0:1773941539.007407:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941539.007409:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03521c00. 00000100:00000001:3.0:1773941539.007412:0:14749:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:2.0:1773941539.007414:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 10000000:01000000:3.0:1773941539.007417:0:14749:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000020:00000001:2.0:1773941539.007417:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941539.007420:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000001:00000001:3.0:1773941539.007423:0:14749:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:2.0:1773941539.007423:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941539.007427:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d035a7400. 00000001:00000001:3.0:1773941539.007429:0:14749:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941539.007430:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941539.007431:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941539.007432:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06506680. 00000100:00000001:2.0:1773941539.007434:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941539.007435:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941539.007436:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941539.007442:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000001:00000001:3.0:1773941539.007443:0:14749:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941539.007444:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941539.007450:0:14749:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:2.0:1773941539.007450:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941539.007465:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941539.007469:0:14749:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941539.007472:0:14749:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941539.007475:0:14749:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941539.007477:0:14749:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941539.007481:0:14749:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:3.0:1773941539.007484:0:14749:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:3.0:1773941539.007486:0:14749:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941539.007488:0:14749:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1773941539.007491:0:14749:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:3.0:1773941539.007493:0:14749:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941539.007494:0:14749:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:3.0:1773941539.007496:0:14749:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:3.0:1773941539.007512:0:14749:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:3.0:1773941539.007514:0:14749:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941539.007516:0:14749:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:3.0:1773941539.007520:0:14749:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:3.0:1773941539.007523:0:14749:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941539.007526:0:14749:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941539.007527:0:14749:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941539.007530:0:14749:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941539.007532:0:14749:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:2.1:1773941539.010997:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d071eec00. 00010000:00000010:2.1:1773941539.011003:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d089de800. 00000020:00000001:3.0:1773941539.835150:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941539.835159:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941539.835165:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941540.283089:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294930601, at 4294930624 00000100:00000010:1.0:1773941540.283110:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d08bbbf00. 00000100:00000040:0.0:1773941540.283110:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294930601, at 4294930624 00000100:00000010:0.0:1773941540.283162:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d08bbb900. 10000000:00000001:2.0:1773941540.411086:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:2.0:1773941540.411096:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941540.411102:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941540.411105:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941540.411107:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941540.411130:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:2.0:1773941540.411134:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:2.0:1773941540.411136:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:2.0:1773941540.411139:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:2.0:1773941540.411154:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:2.0:1773941540.411156:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:2.0:1773941540.411164:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06506680. 00000020:00000040:2.0:1773941540.411170:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941540.411176:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941540.411180:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941540.411187:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941540.411195:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d0381e000. 00000100:00000001:2.0:1773941540.411203:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941540.411208:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:2.0:1773941540.411213:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:2.0:1773941540.411232:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d071eec00. 00010000:00000001:2.0:1773941540.411239:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:2.0:1773941540.411243:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d089de800. 00000020:00000001:2.0:1773941540.411247:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:2.0:1773941540.411251:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d089de800 with handle 0x8e86767622e2998 to hash 00000020:00000001:2.0:1773941540.411253:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:2.0:1773941540.411255:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330235217920 : -113743474333696 : ffff988d089de800) 00010000:00000001:2.0:1773941540.411259:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330235217920 : -113743474333696 : ffff988d089de800) 00010000:00000001:2.0:1773941540.411264:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941540.411267:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941540.411272:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2998 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941540.411285:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2998 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941540.411298:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2998 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941540.411306:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:2.0:1773941540.411313:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:2.0:1773941540.411320:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d09350540. 00000100:00000001:2.0:1773941540.411322:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330245121344 : -113743464430272 : ffff988d09350540) 00000100:00000001:2.0:1773941540.411326:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941540.411328:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584064 : -113743512967552 : ffff988d06506680) 00000100:00000001:2.0:1773941540.411344:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941540.411348:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941540.411351:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:2.0:1773941540.411354:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941540.411365:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06506680 x1860112291485184/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941540.411377:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941540.411380:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941540.411384:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941540.411387:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941540.411390:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941540.411394:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06506680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291485184:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941540.411399:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941540.411404:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941540.411407:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941540.411410:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941540.411415:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a7400. 02000000:00000001:2.0:1773941540.411417:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941540.411447:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941540.411453:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000400:00000010:2.0:1773941540.411460:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000200:2.0:1773941540.411469:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291485184, portal 25 00000100:00000001:2.0:1773941540.411471:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941540.411473:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584064 : -113743512967552 : ffff988d06506680) 00000100:00000040:2.0:1773941540.411479:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06506680 x1860112291485184/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941556 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941540.411488:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941540.411491:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:2.0:1773941540.411500:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516550. 00000100:00000200:2.0:1773941540.411505:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291485184, offset 0 00000400:00000200:2.0:1773941540.411512:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941540.411522:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941540.411531:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941540.411537:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941540.411542:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941540.411547:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941540.411550:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941540.411554:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[68, 67] 00000400:00000200:2.0:1773941540.411558:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:2.0:1773941540.411572:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [179:179:256:4294967295] 192.168.204.124@tcp LPNI seq info [69:69:8:4294967295] 00000400:00000200:2.0:1773941540.411578:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:2.0:1773941540.411590:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:2.0:1773941540.411597:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:2.0:1773941540.411604:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0dbb00. 00000800:00000200:2.0:1773941540.411612:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:2.0:1773941540.411621:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:2.0:1773941540.411626:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0dbb00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:2.0:1773941540.411648:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941540.411653:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941540.411656:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294932007 00000100:00000010:2.0:1773941540.411661:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d08bbbde0. 00000100:00000001:2.0:1773941540.411669:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1773941540.411677:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294932007 00000100:00000010:2.0:1773941540.411681:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d08bbb180. 00000100:00000001:2.0:1773941540.411685:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1773941540.411690:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941540.411693:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941540.411696:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941540.411708:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:2.0:1773941540.411712:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d09350540 going to sleep for 11 seconds 00000100:00000001:2.0:1773941540.411716:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941540.411719:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941540.411867:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941540.411877:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0dbb00. 00000400:00000200:1.0:1773941540.411887:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941540.411897:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941540.411904:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516550 00000400:00000010:1.0:1773941540.411907:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516550. 00000100:00000001:1.0:1773941540.411912:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941540.411919:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06506680 x1860112291485184/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941556 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941540.411933:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941540.411937:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d06506680 x1860112291485184/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941556 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941540.411946:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941540.411949:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941540.413934:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941540.413956:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941540.414010:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941540.414017:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941540.414024:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941540.414032:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941540.414041:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941540.414047:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941540.414051:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941540.414057:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324605600 00000400:00000200:0.0:1773941540.414062:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941540.414069:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xd61 [1] + 224 00000800:00000001:0.0:1773941540.414089:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941540.414099:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941540.414102:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941540.414106:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941540.414111:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941540.414118:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06506680 x1860112291485184/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941556 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941540.414131:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d06506680 x1860112291485184/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941556 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941540.414164:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941540.414172:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941540.414176:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941540.414262:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941540.414271:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941540.414274:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941540.414277:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941540.414283:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941540.414285:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941540.414288:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941540.414297:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06506680 x1860112291485184/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941556 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941540.414315:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06506680 x1860112291485184/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941556 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941540.414324:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941540.414326:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941540.414329:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941540.414332:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941540.414335:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941540.414337:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941540.414340:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941540.414347:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=41 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941540.414352:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941540.414355:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:2.0:1773941540.414359:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:2.0:1773941540.414363:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941540.414367:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:2.0:1773941540.414370:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1773941540.414375:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1773941540.414380:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1773941540.414384:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941540.414386:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941540.414398:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941540.414401:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:2.0:1773941540.414406:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d06506680 x1860112291485184/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941556 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941540.414416:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941540.414417:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941540.414420:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941540.414423:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941540.414448:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:2.0:1773941540.414450:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941540.414453:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941540.414454:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941540.414456:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941540.414458:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941540.414459:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941540.414462:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1773941540.414464:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1773941540.414467:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941540.414471:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 262 00000400:00000200:2.0:1773941540.414477:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941540.414482:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:2.0:1773941540.414487:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941540.414490:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941540.414493:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941540.414496:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:2.0:1773941540.414500:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941540.414502:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941540.414504:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941540.414507:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941540.414509:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941540.414512:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:2.0:1773941540.414515:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941540.414517:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941540.414519:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941540.414527:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:2.0:1773941540.414531:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:2.0:1773941540.414535:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:2.0:1773941540.414538:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941540.414541:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941540.414544:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941540.414547:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941540.414550:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941540.414552:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941540.414556:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06506a00. 00000020:00000040:2.0:1773941540.414559:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:2.0:1773941540.414563:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941540.414565:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941540.414567:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941540.414570:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941540.414573:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941540.414578:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d035a4000. 00000100:00000001:2.0:1773941540.414582:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941540.414589:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d06506a00 x1860112291485312/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941540.414599:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941540.414602:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941540.414608:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6b603d8] to pc [ptlrpcd_rcv+-1] req@ffff988d06506a00 x1860112291485312/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941540.414635:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941540.414639:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:2.0:1773941540.414641:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941540.414644:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941540.414647:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941540.414649:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941540.414651:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d09350540 going to sleep for 1 seconds 00000100:00000001:2.0:1773941540.414654:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941540.414656:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941540.414669:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06506680 x1860112291485184/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941556 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941540.414679:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941540.414683:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941540.414685:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941540.414688:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941540.414690:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1773941540.414696:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06506680 x1860112291485184/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941556 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941540.414698:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941540.414704:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00100000:2.0:1773941540.414720:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06506680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291485184:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000040:3.0:1773941540.414721:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06506a00 x1860112291485312/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941540.414738:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941540.414746:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941540.414748:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941540.414752:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:3.0:1773941540.414754:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941540.414757:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1773941540.414760:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941540.414766:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:2.0:1773941540.414767:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06506680 x1860112291485184/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941556 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1773941540.414774:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941540.414781:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941540.414790:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d09350540. 00000100:00000001:2.0:1773941540.414793:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:2.0:1773941540.414795:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:2.0:1773941540.414802:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00100000:3.0:1773941540.414808:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291485312:192.168.204.114@tcp:250:ll_cfg_requeue.0 00010000:00000001:2.0:1773941540.414809:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:2.0:1773941540.414814:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941540.414820:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089de800 refcount=4 00000100:00000001:3.0:1773941540.414821:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000001:2.0:1773941540.414827:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235217920 : -113743474333696 : ffff988d089de800) 02000000:00000001:3.0:1773941540.414828:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941540.414834:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941540.414835:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235217920 : -113743474333696 : ffff988d089de800) 02000000:00000001:3.0:1773941540.414838:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00010000:2.0:1773941540.414844:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2998 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000010:3.0:1773941540.414857:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03520c00. 02000000:00000001:3.0:1773941540.414865:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941540.414868:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000200:3.0:1773941540.414876:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:2.0:1773941540.414878:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2998 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:3.0:1773941540.414882:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00000400:00000010:3.0:1773941540.414889:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00000100:00000200:3.0:1773941540.414899:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291485312, portal 25 00000100:00000001:3.0:1773941540.414906:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00000001:2.0:1773941540.414906:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000001:3.0:1773941540.414914:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584960 : -113743512966656 : ffff988d06506a00) 00010000:00010000:2.0:1773941540.414915:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2998 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:3.0:1773941540.414924:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06506a00 x1860112291485312/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941550 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1773941540.414931:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941540.414935:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:2.0:1773941540.414941:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2998 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941540.414944:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941540.414954:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00000001:2.0:1773941540.414959:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941540.414997:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941540.415002:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:2.0:1773941540.415006:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2998 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:3.0:1773941540.415011:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac990. 00010000:00010000:2.0:1773941540.415014:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d089de800 already unused, calling callback (ffffffffc12b4c20) 00000100:00000200:3.0:1773941540.415019:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291485312, offset 0 10000000:00000001:2.0:1773941540.415020:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:2.0:1773941540.415026:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2998 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941540.415036:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00000001:2.0:1773941540.415045:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:2.0:1773941540.415049:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:3.0:1773941540.415050:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000020:00000001:2.0:1773941540.415053:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941540.415058:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089de800 refcount=5 00000400:00000200:3.0:1773941540.415062:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000020:00000001:2.0:1773941540.415064:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235217920 : -113743474333696 : ffff988d089de800) 00010000:00000001:2.0:1773941540.415071:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235217920 : -113743474333696 : ffff988d089de800) 00000400:00000200:3.0:1773941540.415072:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941540.415087:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00000001:2.0:1773941540.415088:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:3.0:1773941540.415094:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00010000:2.0:1773941540.415096:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2998 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941540.415101:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941540.415114:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[68, 69] 10000000:00000001:2.0:1773941540.415120:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:3.0:1773941540.415124:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941540.415130:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [180:180:256:4294967295] 192.168.204.114@tcp LPNI seq info [70:70:8:4294967295] 10000000:00010000:2.0:1773941540.415134:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2998 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941540.415141:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 10000000:01000000:2.0:1773941540.415165:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:3.0:1773941540.415166:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 10000000:00000040:2.0:1773941540.415173:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:2.0:1773941540.415176:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941540.415180:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:2.0:1773941540.415182:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:2.0:1773941540.415187:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1773941540.415191:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1773941540.415194:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000800:00000200:3.0:1773941540.415196:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00010000:00000001:2.0:1773941540.415198:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1773941540.415203:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74400. 00000020:00000040:2.0:1773941540.415204:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d089de800 with handle 0x8e86767622e2998 from hash 00010000:00000001:2.0:1773941540.415213:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:2.0:1773941540.415215:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941540.415219:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000200:3.0:1773941540.415220:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00010000:00000001:2.0:1773941540.415222:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:2.0:1773941540.415234:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:2.0:1773941540.415239:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:2.0:1773941540.415243:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000200:3.0:1773941540.415245:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00010000:00000001:2.0:1773941540.415246:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941540.415251:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941540.415254:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74400 type 1, nob 616 niov 1 nkiov 1 10000000:00000001:2.0:1773941540.415256:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941540.415268:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2998 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941540.415274:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941540.415275:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941540.415277:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000100:00000001:3.0:1773941540.415281:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941540.415281:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:2.0:1773941540.415286:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941540.415289:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:3.0:1773941540.415293:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:2.0:1773941540.415293:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:2.0:1773941540.415301:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d089de800/0x8e86767622e2998 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941540.415302:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294932011 00000100:00000001:3.0:1773941540.415315:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294932007 00010000:00000040:2.0:1773941540.415316:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d071eec00 count: 0 00000100:00000001:3.0:1773941540.415334:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294932011 00000100:00000001:3.0:1773941540.415344:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294932007 00010000:00000001:2.0:1773941540.415356:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:3.0:1773941540.415360:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:2.0:1773941540.415361:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941540.415371:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941540.415372:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1773941540.415380:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941540.415384:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06506680 x1860112291485184/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941556 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941540.415386:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941540.415399:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941540.415400:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1773941540.415403:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 02000000:00000001:2.0:1773941540.415406:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1773941540.415410:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:2.0:1773941540.415412:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a7400. 00000800:00000200:1.0:1773941540.415412:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1773941540.415415:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941540.415421:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941540.415423:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000800:00000010:1.0:1773941540.415445:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74400. 00000100:00000001:3.0:1773941540.415446:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941540.415459:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000400:00000200:1.0:1773941540.415461:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000040:2.0:1773941540.415465:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941540.415472:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 00000400:00000200:1.0:1773941540.415475:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 02000000:00000010:2.0:1773941540.415478:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d0381e000. 00000400:00000200:1.0:1773941540.415483:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac990 02000000:00000001:2.0:1773941540.415486:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000400:00000010:1.0:1773941540.415489:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac990. 02000000:00000001:2.0:1773941540.415491:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941540.415494:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06506680. 00000100:00000001:2.0:1773941540.415498:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941540.415501:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:2.0:1773941540.415507:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941540.415510:0:2393:0:(events.c:34:request_out_callback()) Process entered 10000000:01000000:2.0:1773941540.415511:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:2.0:1773941540.415523:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:2.0:1773941540.415531:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d05437ac0. 00000100:00000200:1.0:1773941540.415534:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06506a00 x1860112291485312/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941550 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1773941540.415538:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d0381e000. 00000020:00000010:2.0:1773941540.415548:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d0381e800. 00000040:00000010:2.0:1773941540.415557:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d0381f200. 00000100:00000001:1.0:1773941540.415559:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000020:00000010:2.0:1773941540.415564:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d09305000. 00000100:00000040:1.0:1773941540.415567:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06506a00 x1860112291485312/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941550 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1773941540.415574:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d03356200. 00000100:00000001:1.0:1773941540.415585:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 80000000:00000010:2.0:1773941540.415586:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d08bbb8a0. 00000100:00000001:1.0:1773941540.415591:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00040000:00000010:2.0:1773941540.415596:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d08a4f400. 20000000:00000010:2.0:1773941540.415602:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d08a4fe00. 10000000:00000040:2.0:1773941540.415607:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:2.0:1773941540.415610:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:2.0:1773941540.415616:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d0381e800. 00000040:00000010:2.0:1773941540.415622:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d0381f200. 00000020:00000010:2.0:1773941540.415637:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941540.415642:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d03356200. 80000000:00000010:2.0:1773941540.415645:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d08bbb8a0. 00040000:00000010:2.0:1773941540.415649:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d08a4f400. 20000000:00000010:2.0:1773941540.415653:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d08a4fe00. 00000020:00000010:2.0:1773941540.415657:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0381e000. 10000000:00000010:2.0:1773941540.415659:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d05437ac0. 10000000:01000000:2.0:1773941540.415663:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:2.0:1773941540.415666:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:2.0:1773941540.415669:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:2.0:1773941540.415671:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:2.0:1773941540.415673:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:2.0:1773941540.415674:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:2.0:1773941540.415677:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941540.417886:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941540.417897:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941540.417901:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941540.417905:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941540.417912:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941540.417919:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941540.417929:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941540.417948:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941540.417954:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941540.418005:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324605680 00000400:00000200:0.0:1773941540.418015:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941540.418023:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xd69 [1] + 0 00000800:00000001:0.0:1773941540.418032:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941540.418043:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941540.418047:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941540.418054:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941540.418061:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941540.418070:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06506a00 x1860112291485312/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941550 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941540.418085:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d06506a00 x1860112291485312/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941550 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941540.418105:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941540.418119:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941540.418123:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941540.418174:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941540.418180:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941540.418187:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941540.418200:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941540.418204:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941540.418212:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941540.418216:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941540.418221:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941540.418232:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06506a00 x1860112291485312/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941550 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941540.418252:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06506a00 x1860112291485312/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941550 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941540.418288:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941540.418301:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941540.418305:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941540.418309:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941540.418312:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941540.418315:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941540.418319:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941540.418325:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=41 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941540.418331:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941540.418335:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941540.418338:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941540.418342:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941540.418345:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941540.418350:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941540.418359:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06506a00 x1860112291485312/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941550 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941540.418373:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941540.418377:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941540.418380:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941540.418384:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941540.418392:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941540.418396:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941540.418398:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941540.418402:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941540.418405:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941540.418410:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941540.418414:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941540.418420:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941540 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941540.418452:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941540.418463:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941540.418467:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941540.418470:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941540.418474:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941540.418486:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941540.418493:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941540.418497:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941540.418502:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941540.418507:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941540.418513:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941540.418517:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941540.418876:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941540.418884:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941540.418894:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06506a00 x1860112291485312/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941550 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941540.418908:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291485312:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941540.418912:0:14750:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941540.418916:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941540.418922:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941540.418928:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06506a00 x1860112291485312/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941550 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1773941540.418933:0:14750:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:3.0:1773941540.418948:0:14750:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941540.418948:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941540.418953:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1773941540.418956:0:14750:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000010:2.0:1773941540.418958:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03520c00. 02000000:00000001:2.0:1773941540.418993:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941540.418996:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941540.418998:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941540.419001:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941540.419003:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d035a4000. 02000000:00000001:2.0:1773941540.419007:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 10000000:01000000:3.0:1773941540.419014:0:14750:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000001:2.0:1773941540.419022:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000001:00000001:3.0:1773941540.419025:0:14750:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000010:2.0:1773941540.419027:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06506a00. 00000100:00000001:2.0:1773941540.419034:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941540.419044:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:3.0:1773941540.419046:0:14750:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941540.419048:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:3.0:1773941540.419056:0:14750:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941540.419057:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941540.419062:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941540.419066:0:14750:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:2.0:1773941540.419068:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000001:00000001:3.0:1773941540.419075:0:14750:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941540.419075:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941540.419085:0:14750:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941540.419091:0:14750:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941540.419094:0:14750:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941540.419100:0:14750:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:3.0:1773941540.419105:0:14750:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:3.0:1773941540.419109:0:14750:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941540.419123:0:14750:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1773941540.419127:0:14750:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:3.0:1773941540.419131:0:14750:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941540.419134:0:14750:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:3.0:1773941540.419137:0:14750:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:3.0:1773941540.419161:0:14750:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:3.0:1773941540.419164:0:14750:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941540.419167:0:14750:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:3.0:1773941540.419174:0:14750:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:3.0:1773941540.419183:0:14750:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941540.419189:0:14750:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941540.419203:0:14750:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941540.419208:0:14750:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941540.419220:0:14750:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:2.1:1773941540.421002:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d071eec00. 00010000:00000010:2.1:1773941540.421012:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d089de800. 00000020:00000001:2.0:1773941540.859174:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941540.859182:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941540.859187:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941541.115161:0:461:0:(pinger.c:229:ptlrpc_pinger_process_import()) c1c1ff21-5f68-4bf6-bdc4-258644b24465->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1773941541.115175:0:461:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1773941541.115189:0:461:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c2680. 00000020:00000040:1.0:1773941541.115196:0:461:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941541.115212:0:461:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941541.115216:0:461:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941541.115224:0:461:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941541.115250:0:461:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff988d02cfdd00. 00000100:00000001:1.0:1773941541.115259:0:461:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941541.115275:0:461:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging c1c1ff21-5f68-4bf6-bdc4-258644b24465->MGS req@ffff988d203c2680 x1860112291485440/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941541.115290:0:461:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941541.115295:0:461:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941541.115304:0:461:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000097d1012f] to pc [ptlrpcd_00_01+1] req@ffff988d203c2680 x1860112291485440/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941541.115347:0:461:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941541.115354:0:461:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (269) 00000100:00000001:0.0:1773941541.115431:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941541.115456:0:2402:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941541.115460:0:2402:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941541.115471:0:2402:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c2680 x1860112291485440/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941541.115483:0:2402:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941541.115486:0:2402:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941541.115491:0:2402:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941541.115500:0:2402:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941541.115504:0:2402:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941541.115509:0:2402:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2402:1860112291485440:192.168.204.114@tcp:400:kworker.0 00000100:00000001:0.0:1773941541.115514:0:2402:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941541.115520:0:2402:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941541.115524:0:2402:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941541.115527:0:2402:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941541.115551:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff988d065c4e00. 02000000:00000001:0.0:1773941541.115554:0:2402:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941541.115563:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941541.115568:0:2402:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0014ba50. 00000400:00000010:0.0:1773941541.115575:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d108337f8. 00000100:00000200:0.0:1773941541.115583:0:2402:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112291485440, portal 25 00000100:00000001:0.0:1773941541.115586:0:2402:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941541.115588:0:2402:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631464576 : -113743078087040 : ffff988d203c2680) 00000100:00000040:0.0:1773941541.115596:0:2402:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c2680 x1860112291485440/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941557 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941541.115606:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941541.115609:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941541.115612:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833908. 00000100:00000200:0.0:1773941541.115615:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1860112291485440, offset 0 00000400:00000200:0.0:1773941541.115622:0:2402:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941541.115632:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941541.115641:0:2402:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941541.115647:0:2402:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941541.115652:0:2402:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941541.115667:0:2402:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941541.115671:0:2402:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941541.115675:0:2402:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[70, 69] 00000400:00000200:0.0:1773941541.115680:0:2402:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:0.0:1773941541.115685:0:2402:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [181:181:256:4294967295] 192.168.204.124@tcp LPNI seq info [71:71:8:4294967295] 00000400:00000200:0.0:1773941541.115690:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941541.115702:0:2402:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:0.0:1773941541.115717:0:2402:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:0.0:1773941541.115724:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a20bd00. 00000800:00000200:0.0:1773941541.115732:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:0.0:1773941541.115741:0:2402:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:0.0:1773941541.115747:0:2402:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a20bd00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1773941541.115775:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941541.115781:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941541.115784:0:2402:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294932711 00000100:00000001:0.0:1773941541.115793:0:2402:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294932011 00000100:00000001:0.0:1773941541.115798:0:2402:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294932711 00000100:00000001:0.0:1773941541.115804:0:2402:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294932011 00000100:00000001:0.0:1773941541.115809:0:2402:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941541.115813:0:2402:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941541.115818:0:2402:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941541.115822:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941541.115831:0:2402:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941541.115833:0:2402:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1773941541.115837:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941541.115839:0:2402:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941541.115841:0:2402:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941541.115844:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941541.115860:0:2401:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941541.115863:0:2401:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941541.116043:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941541.116050:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a20bd00. 00000400:00000200:1.0:1773941541.116058:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941541.116066:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941541.116072:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833908 00000400:00000010:1.0:1773941541.116076:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833908. 00000100:00000001:1.0:1773941541.116081:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941541.116097:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c2680 x1860112291485440/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941557 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941541.116109:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941541.116114:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c2680 x1860112291485440/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941557 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941541.116122:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941541.116124:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941541.117946:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941541.118007:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941541.118012:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941541.118016:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941541.118024:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941541.118031:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941541.118041:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941541.118047:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941541.118052:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941541.118058:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324605700 00000400:00000200:0.0:1773941541.118064:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941541.118071:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xd71 [1] + 224 00000800:00000001:0.0:1773941541.118081:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941541.118091:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941541.118094:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941541.118099:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941541.118114:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941541.118122:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c2680 x1860112291485440/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941557 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941541.118136:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff988d203c2680 x1860112291485440/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941557 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941541.118161:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941541.118171:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941541.118176:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941541.118211:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941541.118216:0:2402:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941541.118223:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941541.118226:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941541.118229:0:2402:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0014ba50. 00000400:00000200:1.0:1773941541.118253:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d108337f8 00000400:00000010:1.0:1773941541.118255:0:2402:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d108337f8. 00000100:00000001:1.0:1773941541.118259:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941541.118269:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c2680 x1860112291485440/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941557 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941541.118284:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c2680 x1860112291485440/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941557 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941541.118294:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941541.118296:0:2402:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941541.118300:0:2402:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941541.118303:0:2402:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941541.118306:0:2402:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941541.118309:0:2402:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941541.118312:0:2402:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941541.118348:0:2402:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=42 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941541.118353:0:2402:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941541.118356:0:2402:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1773941541.118359:0:2402:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941541.118363:0:2402:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941541.118367:0:2402:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941541.118372:0:2402:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941541.118377:0:2402:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941541.118394:0:2402:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941541.118434:0:2402:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941541.118438:0:2402:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941541.118442:0:2402:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941541.118449:0:2402:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d203c2680 x1860112291485440/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941557 ref 1 fl Rpc:RNQU/200/0 rc 0/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941541.118459:0:2402:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941541.118461:0:2402:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941541.118463:0:2402:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941541.118467:0:2402:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941541.118470:0:2402:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941541.118472:0:2402:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941541.118474:0:2402:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941541.118476:0:2402:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941541.118477:0:2402:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941541.118479:0:2402:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941541.118480:0:2402:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941541.118483:0:2402:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941541.118486:0:2402:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941541.118489:0:2402:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941541.118492:0:2402:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 263 00000400:00000200:1.0:1773941541.118497:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941541.118503:0:2402:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941541.118507:0:2402:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941541.118510:0:2402:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941541.118514:0:2402:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941541.118516:0:2402:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941541.118520:0:2402:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941541.118522:0:2402:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941541.118525:0:2402:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941541.118528:0:2402:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941541.118531:0:2402:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941541.118533:0:2402:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941541.118545:0:2402:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941541.118549:0:2402:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941541.118552:0:2402:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941541.118556:0:2402:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941541.118561:0:2402:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941541.118566:0:2402:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941541.118570:0:2402:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941541.118574:0:2402:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941541.118579:0:2402:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941541.118584:0:2402:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941541.118587:0:2402:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941541.118606:0:2402:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941541.118612:0:2402:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c2d80. 00000020:00000040:1.0:1773941541.118616:0:2402:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941541.118620:0:2402:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941541.118623:0:2402:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941541.118627:0:2402:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941541.118630:0:2402:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941541.118634:0:2402:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941541.118640:0:2402:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d1118a800. 00000100:00000001:1.0:1773941541.118646:0:2402:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941541.118655:0:2402:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c2d80 x1860112291485568/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941541.118668:0:2402:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941541.118672:0:2402:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941541.118677:0:2402:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c459535b] to pc [ptlrpcd_rcv+-1] req@ffff988d203c2d80 x1860112291485568/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941541.118702:0:2402:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941541.118705:0:2402:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941541.118707:0:2402:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:1.0:1773941541.118712:0:2402:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c2680 x1860112291485440/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941557 ref 1 fl Rpc:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941541.118733:0:2402:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941541.118736:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941541.118739:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941541.118745:0:2402:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c2680 x1860112291485440/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941557 ref 1 fl Interpret:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941541.118754:0:2402:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2402:1860112291485440:192.168.204.114@tcp:400:kworker.0 00000100:00000001:2.0:1773941541.118756:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941541.118760:0:2402:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941541.118765:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941541.118765:0:2402:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941541.118772:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941541.118774:0:2402:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c2680 x1860112291485440/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941557 ref 1 fl Complete:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941541.118784:0:2402:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941541.118789:0:2402:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000040:2.0:1773941541.118791:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c2d80 x1860112291485568/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1773941541.118792:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff988d065c4e00. 02000000:00000001:1.0:1773941541.118800:0:2402:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941541.118804:0:2402:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941541.118809:0:2402:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941541.118816:0:2402:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941541.118822:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 02000000:00000010:1.0:1773941541.118823:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff988d02cfdd00. 00000100:00000001:2.0:1773941541.118829:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941541.118830:0:2402:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941541.118833:0:2402:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941541.118837:0:2402:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c2680. 02000000:00000001:2.0:1773941541.118840:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1773941541.118842:0:2402:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 02000000:00000001:2.0:1773941541.118844:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941541.118846:0:2402:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1773941541.118850:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941541.118852:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941541.118860:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291485568:192.168.204.114@tcp:250:ptlrpcd_00_01.0 00000100:00000001:1.0:1773941541.118861:0:2402:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941541.118865:0:2402:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941541.118870:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1773941541.118871:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1773941541.118875:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1773941541.118876:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941541.118880:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941541.118884:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941541.118890:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a4000. 02000000:00000001:2.0:1773941541.118893:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941541.118898:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941541.118902:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000400:00000010:2.0:1773941541.118906:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000200:2.0:1773941541.118910:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291485568, portal 25 00000100:00000001:2.0:1773941541.118912:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941541.118914:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631466368 : -113743078085248 : ffff988d203c2d80) 00000100:00000040:2.0:1773941541.118930:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c2d80 x1860112291485568/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941551 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941541.118938:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941541.118940:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:2.0:1773941541.118943:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516440. 00000100:00000200:2.0:1773941541.118947:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291485568, offset 0 00000400:00000200:2.0:1773941541.118986:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941541.118992:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941541.118997:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941541.119000:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941541.119003:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941541.119006:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941541.119008:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941541.119011:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[70, 71] 00000400:00000200:2.0:1773941541.119015:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:2.0:1773941541.119018:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [182:182:256:4294967295] 192.168.204.114@tcp LPNI seq info [72:72:8:4294967295] 00000400:00000200:2.0:1773941541.119022:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941541.119029:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:2.0:1773941541.119052:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:2.0:1773941541.119057:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0dbd00. 00000800:00000200:2.0:1773941541.119063:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:2.0:1773941541.119071:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:2.0:1773941541.119075:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0dbd00 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:2.0:1773941541.119094:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941541.119099:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941541.119101:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294932715 00000100:00000001:2.0:1773941541.119108:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294932711 00000100:00000001:2.0:1773941541.119112:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294932715 00000100:00000001:2.0:1773941541.119117:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294932711 00000100:00000001:2.0:1773941541.119122:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941541.119134:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941541.119138:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941541.119142:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941541.119148:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941541.119150:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941541.119153:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941541.119155:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941541.119158:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941541.119161:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941541.119223:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941541.119229:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0dbd00. 00000400:00000200:1.0:1773941541.119235:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941541.119243:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941541.119258:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516440 00000400:00000010:1.0:1773941541.119260:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516440. 00000100:00000001:1.0:1773941541.119264:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941541.119270:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c2d80 x1860112291485568/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941551 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941541.119281:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941541.119284:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c2d80 x1860112291485568/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941551 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941541.119293:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941541.119295:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941541.121174:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941541.121185:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941541.121187:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941541.121190:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941541.121195:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941541.121200:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941541.121208:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941541.121211:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941541.121215:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941541.121219:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324605780 00000400:00000200:0.0:1773941541.121225:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941541.121229:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xd81 [1] + 0 00000800:00000001:0.0:1773941541.121235:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941541.121242:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941541.121245:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941541.121248:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941541.121252:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941541.121258:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c2d80 x1860112291485568/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941551 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941541.121273:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c2d80 x1860112291485568/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941551 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941541.121299:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941541.121307:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941541.121322:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941541.121356:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941541.121362:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941541.121368:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941541.121371:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941541.121375:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941541.121382:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941541.121385:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941541.121389:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941541.121398:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c2d80 x1860112291485568/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941551 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941541.121440:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c2d80 x1860112291485568/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941551 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941541.121452:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941541.121455:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941541.121458:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941541.121461:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941541.121463:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941541.121466:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941541.121469:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941541.121475:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=42 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941541.121480:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941541.121484:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941541.121486:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941541.121491:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941541.121494:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941541.121497:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941541.121505:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c2d80 x1860112291485568/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941551 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941541.121548:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941541.121552:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941541.121555:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941541.121558:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941541.121566:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941541.121569:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941541.121571:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941541.121574:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941541.121577:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941541.121580:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941541.121583:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:2.0:1773941541.121586:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941541 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941541.121590:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941541.121596:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:2.0:1773941541.121598:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941541.121600:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941541.121602:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941541.121604:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941541.121608:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:2.0:1773941541.121610:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941541.121612:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941541.121616:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941541.121619:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:2.0:1773941541.121622:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941541.121890:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941541.121896:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941541.121925:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c2d80 x1860112291485568/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941551 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941541.121942:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291485568:192.168.204.114@tcp:250:ptlrpcd_00_01.0 00000100:00000001:2.0:1773941541.121950:0:14751:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:3.0:1773941541.121951:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941541.121959:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941541.121964:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c2d80 x1860112291485568/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941551 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1773941541.122011:0:14751:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:2.0:1773941541.122016:0:14751:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941541.122019:0:14751:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941541.122022:0:14751:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:2.0:1773941541.122026:0:14751:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941541.122031:0:14751:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:3.0:1773941541.122032:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941541.122038:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941541.122043:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a4000. 00000001:00000001:2.0:1773941541.122048:0:14751:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941541.122054:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:2.0:1773941541.122057:0:14751:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:3.0:1773941541.122062:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:2.0:1773941541.122064:0:14751:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000040:3.0:1773941541.122069:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000001:00000001:2.0:1773941541.122072:0:14751:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941541.122079:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 10000000:00000001:2.0:1773941541.122081:0:14751:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941541.122087:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d1118a800. 00000100:00000001:2.0:1773941541.122089:0:14751:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000001:3.0:1773941541.122101:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00080000:2.0:1773941541.122102:0:14751:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 02000000:00000001:3.0:1773941541.122106:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1773941541.122111:0:14751:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000010:3.0:1773941541.122114:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c2d80. 00000100:00000001:2.0:1773941541.122120:0:14751:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941541.122127:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941541.122128:0:14751:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941541.122134:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1773941541.122137:0:14751:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:3.0:1773941541.122144:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941541.122147:0:14751:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941541.122171:0:14751:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000100:00000001:3.0:1773941541.122173:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 10000000:00000040:2.0:1773941541.122176:0:14751:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00000001:3.0:1773941541.122181:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941541.122189:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941541.122193:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1773941541.122211:0:14751:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:2.0:1773941541.122216:0:14751:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941541.122220:0:14751:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:2.0:1773941541.122229:0:14751:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:2.0:1773941541.122236:0:14751:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941541.122240:0:14751:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941541.122242:0:14751:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941541.122246:0:14751:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941541.122248:0:14751:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941541.883183:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941541.883192:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941541.883197:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:3.0:1773941542.331153:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:3.0:1773941542.331164:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:3.0:1773941542.331170:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941542.331173:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941542.331198:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:3.0:1773941542.331200:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:3.0:1773941542.331204:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:3.0:1773941542.331206:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:3.0:1773941542.331209:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:3.0:1773941542.331212:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:3.0:1773941542.331214:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:3.0:1773941542.331233:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d111f3480. 00000020:00000040:3.0:1773941542.331240:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941542.331247:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941542.331251:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941542.331259:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941542.331269:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d1a1c8c00. 00000100:00000001:3.0:1773941542.331278:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941542.331284:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:3.0:1773941542.331288:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:3.0:1773941542.331297:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d1a01d600. 00010000:00000001:3.0:1773941542.331306:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:3.0:1773941542.331310:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d0168aa00. 00000020:00000010:3.0:1773941542.331318:0:14708:0:(lprocfs_status.c:1408:lprocfs_stats_alloc_one()) alloc '(stats->ls_percpu[cpuid])': 128 at ffff988d03580580. 00000020:00000001:3.0:1773941542.331323:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:3.0:1773941542.331327:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d0168aa00 with handle 0x8e86767622e299f to hash 00000020:00000001:3.0:1773941542.331330:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:3.0:1773941542.331332:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330114288128 : -113743595263488 : ffff988d0168aa00) 00010000:00000001:3.0:1773941542.331336:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330114288128 : -113743595263488 : ffff988d0168aa00) 00010000:00000001:3.0:1773941542.331341:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941542.331345:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941542.331351:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d0168aa00/0x8e86767622e299f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941542.331365:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d0168aa00/0x8e86767622e299f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:3.0:1773941542.331380:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d0168aa00/0x8e86767622e299f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941542.331443:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:3.0:1773941542.331446:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:3.0:1773941542.331456:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1837d300. 00000100:00000001:3.0:1773941542.331459:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330496963328 : -113743212588288 : ffff988d1837d300) 00000100:00000001:3.0:1773941542.331462:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941542.331464:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330377909376 : -113743331642240 : ffff988d111f3480) 00000100:00000001:3.0:1773941542.331470:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941542.331476:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941542.331490:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941542.331492:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941542.331506:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d111f3480 x1860112291485696/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941542.331517:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941542.331520:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941542.331525:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941542.331528:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941542.331531:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941542.331535:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d111f3480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291485696:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941542.331541:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941542.331559:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941542.331563:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941542.331566:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941542.331572:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03523000. 02000000:00000001:3.0:1773941542.331574:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941542.331585:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941542.331590:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00000400:00000010:3.0:1773941542.331600:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00000100:00000200:3.0:1773941542.331610:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291485696, portal 25 00000100:00000001:3.0:1773941542.331612:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941542.331613:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330377909376 : -113743331642240 : ffff988d111f3480) 00000100:00000040:3.0:1773941542.331620:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d111f3480 x1860112291485696/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941558 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941542.331642:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941542.331645:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:3.0:1773941542.331649:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac5d8. 00000100:00000200:3.0:1773941542.331654:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291485696, offset 0 00000400:00000200:3.0:1773941542.331661:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941542.331671:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941542.331681:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941542.331687:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941542.331691:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941542.331695:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941542.331698:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941542.331701:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[72, 71] 00000400:00000200:3.0:1773941542.331705:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:3.0:1773941542.331708:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [183:183:256:4294967295] 192.168.204.124@tcp LPNI seq info [73:73:8:4294967295] 00000400:00000200:3.0:1773941542.331712:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:3.0:1773941542.331721:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:3.0:1773941542.331726:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:3.0:1773941542.331732:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74400. 00000800:00000200:3.0:1773941542.331739:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:3.0:1773941542.331747:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:3.0:1773941542.331752:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74400 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:3.0:1773941542.331772:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941542.331776:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:3.0:1773941542.331779:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294933927 00000100:00000001:3.0:1773941542.331787:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294932715 00000100:00000001:3.0:1773941542.331792:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294933927 00000100:00000001:3.0:1773941542.331795:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294932715 00000100:00000001:3.0:1773941542.331798:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941542.331801:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941542.331808:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941542.331810:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:3.0:1773941542.331813:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1837d300 going to sleep for 11 seconds 00000100:00000001:3.0:1773941542.331816:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941542.331819:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941542.331907:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941542.331913:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74400. 00000400:00000200:1.0:1773941542.331918:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941542.331923:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941542.331930:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac5d8 00000400:00000010:1.0:1773941542.331932:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac5d8. 00000100:00000001:1.0:1773941542.331936:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941542.331940:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d111f3480 x1860112291485696/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941558 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941542.331950:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941542.331952:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d111f3480 x1860112291485696/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941558 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941542.331958:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941542.331996:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941542.333053:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941542.333065:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941542.333068:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941542.333069:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941542.333075:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941542.333080:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941542.333086:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941542.333090:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941542.333094:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941542.333098:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324605800 00000400:00000200:0.0:1773941542.333101:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941542.333104:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xd89 [1] + 224 00000800:00000001:0.0:1773941542.333111:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941542.333118:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941542.333120:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941542.333122:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941542.333126:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941542.333130:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d111f3480 x1860112291485696/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941558 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941542.333139:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d111f3480 x1860112291485696/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941558 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941542.333155:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941542.333162:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941542.333164:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941542.333181:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941542.333194:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941542.333196:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941542.333198:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941542.333203:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941542.333204:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941542.333206:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941542.333211:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d111f3480 x1860112291485696/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941558 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941542.333220:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d111f3480 x1860112291485696/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941558 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941542.333226:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941542.333227:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941542.333229:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941542.333232:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941542.333234:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941542.333235:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941542.333238:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941542.333244:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=43 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941542.333246:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941542.333249:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:3.0:1773941542.333252:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:3.0:1773941542.333254:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941542.333257:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:3.0:1773941542.333259:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:3.0:1773941542.333263:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1773941542.333266:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1773941542.333269:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:3.0:1773941542.333277:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941542.333279:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:3.0:1773941542.333280:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:3.0:1773941542.333284:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d111f3480 x1860112291485696/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941558 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941542.333290:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941542.333291:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941542.333292:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941542.333295:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1773941542.333297:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:3.0:1773941542.333298:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941542.333300:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941542.333300:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941542.333301:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:3.0:1773941542.333302:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941542.333303:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941542.333306:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1773941542.333308:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1773941542.333311:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1773941542.333313:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 264 00000400:00000200:3.0:1773941542.333317:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941542.333320:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:3.0:1773941542.333323:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:3.0:1773941542.333326:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941542.333328:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:3.0:1773941542.333330:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:3.0:1773941542.333333:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941542.333335:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941542.333337:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941542.333338:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941542.333340:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941542.333341:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:3.0:1773941542.333343:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:3.0:1773941542.333348:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941542.333350:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:3.0:1773941542.333351:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:3.0:1773941542.333354:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:3.0:1773941542.333357:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:3.0:1773941542.333359:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941542.333361:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1773941542.333363:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941542.333365:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941542.333367:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:3.0:1773941542.333368:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941542.333371:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d111f0a80. 00000020:00000040:3.0:1773941542.333373:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:3.0:1773941542.333376:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1773941542.333377:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941542.333379:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941542.333380:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941542.333382:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941542.333385:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d03522800. 00000100:00000001:3.0:1773941542.333389:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941542.333411:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d111f0a80 x1860112291485824/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1773941542.333418:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941542.333420:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941542.333423:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b933c3a] to pc [ptlrpcd_rcv+-1] req@ffff988d111f0a80 x1860112291485824/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941542.333440:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941542.333442:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:3.0:1773941542.333444:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941542.333446:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941542.333448:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941542.333449:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1773941542.333455:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1837d300 going to sleep for 1 seconds 00000100:00000001:3.0:1773941542.333457:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941542.333459:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941542.333463:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d111f3480 x1860112291485696/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941558 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941542.333468:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941542.333470:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941542.333473:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941542.333474:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941542.333476:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1773941542.333478:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941542.333481:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941542.333485:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d111f0a80 x1860112291485824/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941542.333489:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d111f3480 x1860112291485696/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941558 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941542.333507:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:3.0:1773941542.333510:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d111f3480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291485696:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941542.333510:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941542.333514:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1773941542.333516:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1773941542.333517:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941542.333519:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:2.0:1773941542.333521:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941542.333523:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:3.0:1773941542.333526:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00100000:2.0:1773941542.333527:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d111f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291485824:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000040:3.0:1773941542.333533:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d111f3480 x1860112291485696/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941558 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941542.333534:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1773941542.333539:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941542.333541:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1837d300. 00000100:00000001:3.0:1773941542.333545:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 02000000:00000001:2.0:1773941542.333546:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1773941542.333547:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:2.0:1773941542.333548:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941542.333551:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 02000000:00000001:2.0:1773941542.333551:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:3.0:1773941542.333555:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 02000000:00000010:2.0:1773941542.333557:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a7400. 00000020:00000001:3.0:1773941542.333558:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941542.333562:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d0168aa00 refcount=4 02000000:00000001:2.0:1773941542.333562:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941542.333566:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330114288128 : -113743595263488 : ffff988d0168aa00) 00000400:00000200:2.0:1773941542.333570:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:3.0:1773941542.333571:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330114288128 : -113743595263488 : ffff988d0168aa00) 00000400:00000010:2.0:1773941542.333575:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00010000:00010000:3.0:1773941542.333576:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d0168aa00/0x8e86767622e299f lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941542.333579:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000200:2.0:1773941542.333585:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291485824, portal 25 00000100:00000001:2.0:1773941542.333587:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941542.333588:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330377898624 : -113743331652992 : ffff988d111f0a80) 00000100:00000040:2.0:1773941542.333592:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d111f0a80 x1860112291485824/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941552 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941542.333600:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941542.333606:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:2.0:1773941542.333610:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045164c8. 00010000:00000001:3.0:1773941542.333612:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000200:2.0:1773941542.333614:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291485824, offset 0 00010000:00010000:3.0:1773941542.333619:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d0168aa00/0x8e86767622e299f lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941542.333623:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941542.333629:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:3.0:1773941542.333630:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:3.0:1773941542.333634:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d0168aa00/0x8e86767622e299f lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941542.333634:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941542.333642:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:3.0:1773941542.333647:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:2.0:1773941542.333648:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00000001:3.0:1773941542.333651:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:2.0:1773941542.333652:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00010000:3.0:1773941542.333655:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d0168aa00/0x8e86767622e299f lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941542.333655:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941542.333662:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[72, 73] 00010000:00000001:3.0:1773941542.333670:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:2.0:1773941542.333671:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00010000:00000001:3.0:1773941542.333673:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941542.333678:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [184:184:256:4294967295] 192.168.204.114@tcp LPNI seq info [74:74:8:4294967295] 00010000:00000001:3.0:1773941542.333679:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:3.0:1773941542.333682:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d0168aa00/0x8e86767622e299f lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941542.333685:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:3.0:1773941542.333693:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d0168aa00 already unused, calling callback (ffffffffc12b4c20) 00000400:00000200:2.0:1773941542.333695:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 10000000:00000001:3.0:1773941542.333697:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:3.0:1773941542.333701:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d0168aa00/0x8e86767622e299f lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941542.333702:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:2.0:1773941542.333710:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0db000. 00010000:00000001:3.0:1773941542.333711:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:3.0:1773941542.333714:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1773941542.333716:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941542.333718:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d0168aa00 refcount=5 00000020:00000001:3.0:1773941542.333723:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330114288128 : -113743595263488 : ffff988d0168aa00) 00000800:00000200:2.0:1773941542.333723:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00010000:00000001:3.0:1773941542.333725:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330114288128 : -113743595263488 : ffff988d0168aa00) 00010000:00000001:3.0:1773941542.333728:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:3.0:1773941542.333737:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d0168aa00/0x8e86767622e299f lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941542.333743:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 10000000:00000001:3.0:1773941542.333745:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:3.0:1773941542.333749:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d0168aa00/0x8e86767622e299f lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941542.333750:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0db000 type 1, nob 616 niov 1 nkiov 1 10000000:01000000:3.0:1773941542.333759:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:3.0:1773941542.333762:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:3.0:1773941542.333763:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941542.333765:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:3.0:1773941542.333766:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:3.0:1773941542.333769:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000100:00000001:2.0:1773941542.333770:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941542.333771:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:3.0:1773941542.333776:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:2.0:1773941542.333780:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:3.0:1773941542.333781:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941542.333784:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294933929 00000020:00000040:3.0:1773941542.333786:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d0168aa00 with handle 0x8e86767622e299f from hash 00000100:00000001:2.0:1773941542.333790:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294933927 00010000:00000001:3.0:1773941542.333792:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000100:00000001:2.0:1773941542.333795:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294933929 00010000:00000001:3.0:1773941542.333796:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941542.333801:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941542.333801:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294933927 00010000:00000001:3.0:1773941542.333804:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000100:00000001:2.0:1773941542.333806:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:3.0:1773941542.333809:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000100:00000001:2.0:1773941542.333809:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941542.333814:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941542.333815:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000100:00000001:2.0:1773941542.333817:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1773941542.333821:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941542.333823:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941542.333824:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941542.333825:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 10000000:00000001:3.0:1773941542.333826:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941542.333828:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00010000:00010000:3.0:1773941542.333831:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d0168aa00/0x8e86767622e299f lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941542.333832:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941542.333835:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000800:00000200:1.0:1773941542.333837:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1773941542.333838:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:1.0:1773941542.333843:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0db000. 00010000:00000001:3.0:1773941542.333844:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941542.333844:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941542.333848:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941542.333851:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000400:00000200:1.0:1773941542.333852:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000001:3.0:1773941542.333853:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:3.0:1773941542.333856:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941542.333858:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:1.0:1773941542.333861:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941542.333864:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045164c8 00010000:00000001:3.0:1773941542.333868:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000010:1.0:1773941542.333869:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045164c8. 00010000:00010000:3.0:1773941542.333872:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d0168aa00/0x8e86767622e299f lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941542.333873:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941542.333879:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d111f0a80 x1860112291485824/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941552 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941542.333899:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941542.333902:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d111f0a80 x1860112291485824/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941552 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941542.333907:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941542.333908:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000040:3.0:1773941542.333976:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d1a01d600 count: 0 00010000:00000001:3.0:1773941542.333989:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941542.333990:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941542.333993:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941542.333997:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d111f3480 x1860112291485696/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941558 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941542.334003:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941542.334005:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941542.334008:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03523000. 02000000:00000001:3.0:1773941542.334011:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941542.334013:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941542.334015:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941542.334016:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941542.334019:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d1a1c8c00. 02000000:00000001:3.0:1773941542.334022:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941542.334023:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941542.334025:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d111f3480. 00000100:00000001:3.0:1773941542.334028:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941542.334029:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:3.0:1773941542.334030:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:3.0:1773941542.334032:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:3.0:1773941542.334034:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:3.0:1773941542.334038:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d03ec7e40. 00000020:00000010:3.0:1773941542.334042:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d1a1c8c00. 00000020:00000010:3.0:1773941542.334046:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d1a1c8800. 00000040:00000010:3.0:1773941542.334049:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d1a1c9800. 00000020:00000010:3.0:1773941542.334054:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d08995000. 00000020:00000010:3.0:1773941542.334058:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d1a1c8200. 80000000:00000010:3.0:1773941542.334062:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d111f7c00. 00040000:00000010:3.0:1773941542.334066:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d1a1c8600. 20000000:00000010:3.0:1773941542.334071:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d1a1c8e00. 10000000:00000040:3.0:1773941542.334083:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:3.0:1773941542.334086:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:3.0:1773941542.334092:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d1a1c8800. 00000040:00000010:3.0:1773941542.334096:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d1a1c9800. 00000020:00000010:3.0:1773941542.334099:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d08995000. 00000020:00000010:3.0:1773941542.334116:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d1a1c8200. 80000000:00000010:3.0:1773941542.334119:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d111f7c00. 00040000:00000010:3.0:1773941542.334123:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d1a1c8600. 20000000:00000010:3.0:1773941542.334127:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d1a1c8e00. 00000020:00000010:3.0:1773941542.334130:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d1a1c8c00. 10000000:00000010:3.0:1773941542.334133:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d03ec7e40. 10000000:01000000:3.0:1773941542.334138:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:3.0:1773941542.334142:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:3.0:1773941542.334144:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:3.0:1773941542.334145:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:3.0:1773941542.334146:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:3.0:1773941542.334148:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:3.0:1773941542.334150:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941542.335284:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941542.335292:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941542.335295:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941542.335298:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941542.335303:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941542.335307:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941542.335319:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941542.335322:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941542.335325:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941542.335328:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324605880 00000400:00000200:0.0:1773941542.335331:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941542.335334:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xd91 [1] + 0 00000800:00000001:0.0:1773941542.335337:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941542.335342:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941542.335344:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941542.335347:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941542.335350:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941542.335353:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d111f0a80 x1860112291485824/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941552 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941542.335362:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d111f0a80 x1860112291485824/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941552 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941542.335378:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941542.335382:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941542.335388:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941542.335427:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941542.335431:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941542.335443:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941542.335445:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941542.335447:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941542.335452:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941542.335454:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941542.335457:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941542.335463:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d111f0a80 x1860112291485824/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941552 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941542.335474:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d111f0a80 x1860112291485824/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941552 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941542.335482:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941542.335484:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941542.335486:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941542.335494:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941542.335495:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941542.335507:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941542.335509:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941542.335512:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=43 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941542.335514:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941542.335516:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941542.335517:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941542.335519:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941542.335521:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941542.335523:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941542.335526:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d111f0a80 x1860112291485824/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941552 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941542.335531:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941542.335533:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941542.335534:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941542.335536:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941542.335540:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941542.335542:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941542.335543:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941542.335544:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941542.335546:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941542.335547:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941542.335549:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:2.0:1773941542.335551:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941542 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941542.335553:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941542.335559:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:2.0:1773941542.335561:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941542.335562:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941542.335563:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941542.335565:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941542.335567:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:2.0:1773941542.335569:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941542.335571:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941542.335578:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941542.335581:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:2.0:1773941542.335583:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941542.335834:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941542.335837:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941542.335842:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d111f0a80 x1860112291485824/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941552 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941542.335850:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d111f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291485824:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941542.335855:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941542.335855:0:14753:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:3.0:1773941542.335859:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941542.335862:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d111f0a80 x1860112291485824/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941552 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1773941542.335866:0:14753:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00000001:3.0:1773941542.335873:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00100000:2.0:1773941542.335874:0:14753:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000001:3.0:1773941542.335875:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1773941542.335877:0:14753:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000010:3.0:1773941542.335878:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a7400. 10000000:01000000:2.0:1773941542.335880:0:14753:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000001:3.0:1773941542.335883:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:2.0:1773941542.335884:0:14753:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941542.335886:0:14753:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941542.335889:0:14753:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941542.335890:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:2.0:1773941542.335892:0:14753:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000040:3.0:1773941542.335893:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000001:00000001:2.0:1773941542.335895:0:14753:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000001:3.0:1773941542.335897:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:2.0:1773941542.335898:0:14753:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941542.335900:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d03522800. 10000000:00000001:2.0:1773941542.335901:0:14753:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941542.335905:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1773941542.335905:0:14753:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000001:3.0:1773941542.335908:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:2.0:1773941542.335909:0:14753:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000010:3.0:1773941542.335910:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d111f0a80. 00000100:00000001:2.0:1773941542.335915:0:14753:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:3.0:1773941542.335916:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941542.335918:0:14753:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941542.335919:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941542.335921:0:14753:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941542.335922:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1773941542.335924:0:14753:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:3.0:1773941542.335927:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941542.335928:0:14753:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:3.0:1773941542.335931:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 10000000:01000000:2.0:1773941542.335931:0:14753:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:2.0:1773941542.335935:0:14753:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00000001:3.0:1773941542.335936:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941542.335938:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1773941542.335951:0:14753:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:2.0:1773941542.335953:0:14753:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941542.335954:0:14753:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:2.0:1773941542.335956:0:14753:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:2.0:1773941542.335983:0:14753:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941542.335986:0:14753:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941542.335987:0:14753:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941542.335990:0:14753:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941542.335991:0:14753:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:3.1:1773941542.340988:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d1a01d600. 00010000:00000010:3.1:1773941542.340996:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d0168aa00. 00000020:00000001:2.0:1773941542.907198:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941542.907204:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941542.907223:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1773941543.611084:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294933929, at 4294933952 00000100:00000040:1.0:1773941543.611090:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294933929, at 4294933952 00000100:00000010:0.0:1773941543.611103:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d08bbbde0. 00000100:00000010:1.0:1773941543.611109:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d08bbb180. 00000400:00000010:0.0:1773941543.867192:0:2398:0:(lib-lnet.h:485:lnet_rspt_free()) free 'rspt': 56 at ffff988d09a24c78. 00000400:00000010:0.0:1773941543.867226:0:2398:0:(lib-lnet.h:485:lnet_rspt_free()) free 'rspt': 56 at ffff988d09a24380. 00000020:00000001:0.0:1773941543.931108:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941543.931132:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941543.931136:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:0.0:1773941544.315092:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1773941544.315103:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941544.315109:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941544.315113:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941544.315115:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941544.315117:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1773941544.315122:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:0.0:1773941544.315124:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:0.0:1773941544.315150:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:0.0:1773941544.315154:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1773941544.315156:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:0.0:1773941544.315166:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10837480. 00000020:00000040:0.0:1773941544.315173:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941544.315179:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941544.315184:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941544.315191:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941544.315202:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065c5800. 00000100:00000001:0.0:1773941544.315211:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941544.315217:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1773941544.315221:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1773941544.315231:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d21ec1400. 00010000:00000001:0.0:1773941544.315240:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1773941544.315244:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d00030600. 00000020:00000001:0.0:1773941544.315250:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:0.0:1773941544.315253:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d00030600 with handle 0x8e86767622e29a6 to hash 00000020:00000001:0.0:1773941544.315256:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:0.0:1773941544.315258:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330090849792 : -113743618701824 : ffff988d00030600) 00010000:00000001:0.0:1773941544.315262:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330090849792 : -113743618701824 : ffff988d00030600) 00010000:00000001:0.0:1773941544.315267:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941544.315271:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941544.315276:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e29a6 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941544.315289:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e29a6 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941544.315319:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e29a6 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941544.315327:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1773941544.315329:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941544.315338:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb6a80. 00000100:00000001:0.0:1773941544.315346:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330139634304 : -113743569917312 : ffff988d02eb6a80) 00000100:00000001:0.0:1773941544.315350:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941544.315380:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367702144 : -113743341849472 : ffff988d10837480) 00000100:00000001:0.0:1773941544.315387:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941544.315393:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941544.315396:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941544.315398:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941544.315412:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10837480 x1860112291485952/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941544.315423:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941544.315426:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941544.315431:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941544.315434:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941544.315437:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941544.315441:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10837480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291485952:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941544.315446:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941544.315453:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941544.315456:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941544.315460:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941544.315466:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d02ce4800. 02000000:00000001:0.0:1773941544.315469:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941544.315479:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941544.315485:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0014ba50. 00000400:00000010:0.0:1773941544.315494:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833dd0. 00000100:00000200:0.0:1773941544.315504:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291485952, portal 25 00000100:00000001:0.0:1773941544.315506:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941544.315508:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367702144 : -113743341849472 : ffff988d10837480) 00000100:00000040:0.0:1773941544.315515:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10837480 x1860112291485952/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941560 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941544.315524:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941544.315528:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941544.315531:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833aa0. 00000100:00000200:0.0:1773941544.315534:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291485952, offset 0 00000400:00000200:0.0:1773941544.315549:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941544.315560:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941544.315570:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941544.315576:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941544.315582:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941544.315587:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941544.315591:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941544.315595:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[74, 73] 00000400:00000200:0.0:1773941544.315599:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:0.0:1773941544.315604:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [185:185:256:4294967295] 192.168.204.124@tcp LPNI seq info [75:75:8:4294967295] 00000400:00000200:0.0:1773941544.315609:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941544.315622:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:0.0:1773941544.315631:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:0.0:1773941544.315638:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a20bd00. 00000800:00000200:0.0:1773941544.315647:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:0.0:1773941544.315658:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:0.0:1773941544.315664:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a20bd00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1773941544.315686:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941544.315691:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941544.315694:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294935911 00000100:00000010:0.0:1773941544.315700:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580b40. 00000100:00000001:0.0:1773941544.315709:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1773941544.315719:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294935911 00000100:00000010:0.0:1773941544.315724:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580e40. 00000100:00000001:0.0:1773941544.315729:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1773941544.315734:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941544.315738:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941544.315741:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941544.315743:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1773941544.315746:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6a80 going to sleep for 11 seconds 00000100:00000001:0.0:1773941544.315751:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941544.315760:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941544.315853:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941544.315861:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a20bd00. 00000400:00000200:1.0:1773941544.315866:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941544.315873:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941544.315879:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833aa0 00000400:00000010:1.0:1773941544.315882:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833aa0. 00000100:00000001:1.0:1773941544.315885:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941544.315892:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10837480 x1860112291485952/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941560 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941544.315902:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941544.315904:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d10837480 x1860112291485952/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941560 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941544.315910:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941544.315911:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941544.316982:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941544.316999:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941544.317002:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941544.317006:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941544.317012:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941544.317019:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941544.317027:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941544.317035:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941544.317040:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941544.317046:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324605900 00000400:00000200:0.0:1773941544.317051:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941544.317057:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xd99 [1] + 224 00000800:00000001:0.0:1773941544.317065:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941544.317078:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941544.317082:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941544.317087:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941544.317094:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941544.317100:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10837480 x1860112291485952/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941560 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941544.317113:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d10837480 x1860112291485952/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941560 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941544.317137:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941544.317145:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941544.317150:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941544.317172:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941544.317180:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941544.317182:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941544.317184:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0014ba50. 00000400:00000200:3.0:1773941544.317188:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833dd0 00000400:00000010:3.0:1773941544.317189:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833dd0. 00000100:00000001:3.0:1773941544.317192:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941544.317197:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10837480 x1860112291485952/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941560 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941544.317206:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10837480 x1860112291485952/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941560 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941544.317211:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941544.317212:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941544.317214:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941544.317217:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941544.317219:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941544.317221:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941544.317223:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941544.317229:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=45 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941544.317232:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941544.317235:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:3.0:1773941544.317238:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:3.0:1773941544.317240:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941544.317256:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:3.0:1773941544.317258:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:3.0:1773941544.317262:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1773941544.317265:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1773941544.317268:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:3.0:1773941544.317270:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941544.317271:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:3.0:1773941544.317273:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:3.0:1773941544.317276:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d10837480 x1860112291485952/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941560 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941544.317281:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941544.317282:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941544.317283:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941544.317286:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1773941544.317289:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:3.0:1773941544.317290:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941544.317291:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941544.317292:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941544.317292:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:3.0:1773941544.317293:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941544.317294:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941544.317296:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1773941544.317297:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1773941544.317299:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1773941544.317302:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 265 00000400:00000200:3.0:1773941544.317307:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941544.317311:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:3.0:1773941544.317314:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:3.0:1773941544.317317:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941544.317319:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:3.0:1773941544.317320:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:3.0:1773941544.317323:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941544.317328:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941544.317331:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941544.317333:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941544.317334:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941544.317336:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:3.0:1773941544.317338:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:3.0:1773941544.317340:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941544.317341:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:3.0:1773941544.317342:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:3.0:1773941544.317344:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:3.0:1773941544.317346:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:3.0:1773941544.317348:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941544.317350:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1773941544.317365:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941544.317368:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941544.317370:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:3.0:1773941544.317372:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941544.317376:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d111f0a80. 00000020:00000040:3.0:1773941544.317379:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:3.0:1773941544.317382:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1773941544.317383:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941544.317386:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941544.317388:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941544.317391:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941544.317395:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d03522800. 00000100:00000001:3.0:1773941544.317400:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941544.317405:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d111f0a80 x1860112291486080/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1773941544.317411:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941544.317415:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941544.317418:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b933c3a] to pc [ptlrpcd_rcv+-1] req@ffff988d111f0a80 x1860112291486080/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941544.317444:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941544.317446:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:3.0:1773941544.317447:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941544.317450:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941544.317452:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941544.317453:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1773941544.317455:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6a80 going to sleep for 1 seconds 00000100:00000001:3.0:1773941544.317456:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941544.317458:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941544.317461:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10837480 x1860112291485952/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941560 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941544.317466:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941544.317469:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941544.317471:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941544.317473:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941544.317476:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10837480 x1860112291485952/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941560 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941544.317480:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:3.0:1773941544.317482:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10837480 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291485952:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941544.317485:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941544.317487:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941544.317488:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1773941544.317490:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941544.317494:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:3.0:1773941544.317496:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941544.317497:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d111f0a80 x1860112291486080/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941544.317502:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10837480 x1860112291485952/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941560 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941544.317518:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941544.317522:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941544.317522:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941544.317527:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb6a80. 02000000:00000001:2.0:1773941544.317528:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1773941544.317530:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 02000000:00000001:2.0:1773941544.317531:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941544.317533:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:2.0:1773941544.317534:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941544.317538:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00100000:2.0:1773941544.317540:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d111f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291486080:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941544.317543:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941544.317547:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00000001:3.0:1773941544.317548:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 02000000:00000001:2.0:1773941544.317549:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941544.317552:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:2.0:1773941544.317553:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:3.0:1773941544.317558:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030600 refcount=4 02000000:00000010:2.0:1773941544.317559:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a4c00. 00000020:00000001:3.0:1773941544.317563:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090849792 : -113743618701824 : ffff988d00030600) 02000000:00000001:2.0:1773941544.317564:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941544.317570:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090849792 : -113743618701824 : ffff988d00030600) 00000400:00000200:2.0:1773941544.317570:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941544.317575:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00010000:00010000:3.0:1773941544.317579:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e29a6 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941544.317580:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000200:2.0:1773941544.317586:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291486080, portal 25 00000100:00000001:2.0:1773941544.317590:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941544.317593:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330377898624 : -113743331652992 : ffff988d111f0a80) 00010000:00000001:3.0:1773941544.317595:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000040:2.0:1773941544.317599:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d111f0a80 x1860112291486080/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941554 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:3.0:1773941544.317602:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e29a6 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941544.317607:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941544.317613:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:2.0:1773941544.317618:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045164c8. 00010000:00000001:3.0:1773941544.317619:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000200:2.0:1773941544.317622:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291486080, offset 0 00010000:00010000:3.0:1773941544.317624:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e29a6 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941544.317629:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941544.317636:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:3.0:1773941544.317638:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941544.317642:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:2.0:1773941544.317643:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00010000:3.0:1773941544.317647:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e29a6 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941544.317658:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:2.0:1773941544.317658:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:3.0:1773941544.317661:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941544.317664:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00000001:3.0:1773941544.317667:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:2.0:1773941544.317669:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00010000:3.0:1773941544.317671:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e29a6 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941544.317674:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941544.317682:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[74, 75] 00010000:00010000:3.0:1773941544.317685:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d00030600 already unused, calling callback (ffffffffc12b4c20) 00000400:00000200:2.0:1773941544.317688:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 10000000:00000001:3.0:1773941544.317689:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:3.0:1773941544.317694:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e29a6 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941544.317695:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [186:186:256:4294967295] 192.168.204.114@tcp LPNI seq info [76:76:8:4294967295] 00000400:00000200:2.0:1773941544.317706:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:3.0:1773941544.317710:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000400:00000200:2.0:1773941544.317716:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00010000:00000001:3.0:1773941544.317721:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000800:00000200:2.0:1773941544.317722:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000020:00000001:3.0:1773941544.317724:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941544.317727:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030600 refcount=5 00000800:00000010:2.0:1773941544.317728:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0db900. 00000020:00000001:3.0:1773941544.317729:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090849792 : -113743618701824 : ffff988d00030600) 00010000:00000001:3.0:1773941544.317734:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090849792 : -113743618701824 : ffff988d00030600) 00010000:00000001:3.0:1773941544.317736:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:3.0:1773941544.317740:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e29a6 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941544.317741:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 10000000:00000001:3.0:1773941544.317749:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:3.0:1773941544.317751:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e29a6 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941544.317756:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 10000000:01000000:3.0:1773941544.317759:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:3.0:1773941544.317764:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:3.0:1773941544.317765:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941544.317765:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0db900 type 1, nob 616 niov 1 nkiov 1 00010000:00010000:3.0:1773941544.317769:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:3.0:1773941544.317772:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:3.0:1773941544.317774:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:3.0:1773941544.317776:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:3.0:1773941544.317777:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941544.317779:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1773941544.317781:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d00030600 with handle 0x8e86767622e29a6 from hash 00010000:00000001:3.0:1773941544.317782:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:3.0:1773941544.317783:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941544.317784:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941544.317786:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000100:00000001:2.0:1773941544.317787:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941544.317788:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:3.0:1773941544.317791:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:3.0:1773941544.317794:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:2.0:1773941544.317796:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:3.0:1773941544.317797:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941544.317799:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941544.317802:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941544.317803:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294935913 00010000:00010000:3.0:1773941544.317804:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e29a6 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941544.317814:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294935911 00010000:00000001:3.0:1773941544.317818:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941544.317820:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941544.317821:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000100:00000001:2.0:1773941544.317823:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294935913 00010000:00000001:3.0:1773941544.317824:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:3.0:1773941544.317826:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941544.317829:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941544.317831:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941544.317832:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294935911 00010000:00010000:3.0:1773941544.317833:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d00030600/0x8e86767622e29a6 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941544.317841:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000040:3.0:1773941544.317843:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d21ec1400 count: 0 00000100:00000001:2.0:1773941544.317847:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941544.317853:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941544.317856:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1773941544.317859:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941544.317861:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941544.317864:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941544.317864:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:3.0:1773941544.317869:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10837480 x1860112291485952/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941560 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941544.317869:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941544.317876:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941544.317879:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941544.317881:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1773941544.317883:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941544.317883:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 02000000:00000001:3.0:1773941544.317884:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941544.317889:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d02ce4800. 00000800:00000010:1.0:1773941544.317889:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0db900. 00000100:00000001:2.0:1773941544.317890:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941544.317896:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000400:00000200:1.0:1773941544.317896:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:3.0:1773941544.317900:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941544.317902:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000400:00000200:1.0:1773941544.317905:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000020:00000001:3.0:1773941544.317906:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 00000400:00000200:1.0:1773941544.317911:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045164c8 02000000:00000010:3.0:1773941544.317912:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065c5800. 00000400:00000010:1.0:1773941544.317916:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045164c8. 02000000:00000001:3.0:1773941544.317918:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941544.317921:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1773941544.317921:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000010:3.0:1773941544.317923:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10837480. 00000100:00000200:1.0:1773941544.317927:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d111f0a80 x1860112291486080/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941554 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941544.317935:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941544.317944:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d111f0a80 x1860112291486080/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941554 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941544.317949:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941544.317950:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1773941544.318011:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941544.318012:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:3.0:1773941544.318014:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:3.0:1773941544.318016:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:3.0:1773941544.318018:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:3.0:1773941544.318022:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d03ec7e40. 00000020:00000010:3.0:1773941544.318026:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d1a1c8c00. 00000020:00000010:3.0:1773941544.318034:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d1a1c8e00. 00000040:00000010:3.0:1773941544.318037:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d1a1c8600. 00000020:00000010:3.0:1773941544.318041:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d08995000. 00000020:00000010:3.0:1773941544.318044:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d1a1c8200. 80000000:00000010:3.0:1773941544.318047:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d111f7c00. 00040000:00000010:3.0:1773941544.318051:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d1a1c9800. 20000000:00000010:3.0:1773941544.318054:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d1a1c8800. 10000000:00000040:3.0:1773941544.318058:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:3.0:1773941544.318059:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:3.0:1773941544.318063:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d1a1c8e00. 00000040:00000010:3.0:1773941544.318065:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d1a1c8600. 00000020:00000010:3.0:1773941544.318067:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d08995000. 00000020:00000010:3.0:1773941544.318071:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d1a1c8200. 80000000:00000010:3.0:1773941544.318073:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d111f7c00. 00040000:00000010:3.0:1773941544.318075:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d1a1c9800. 20000000:00000010:3.0:1773941544.318077:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d1a1c8800. 00000020:00000010:3.0:1773941544.318079:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d1a1c8c00. 10000000:00000010:3.0:1773941544.318081:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d03ec7e40. 10000000:01000000:3.0:1773941544.318084:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:3.0:1773941544.318086:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:3.0:1773941544.318087:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:3.0:1773941544.318089:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:3.0:1773941544.318091:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:3.0:1773941544.318092:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:3.0:1773941544.318094:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941544.319104:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941544.319114:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941544.319117:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941544.319120:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941544.319126:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941544.319132:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941544.319139:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941544.319143:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941544.319155:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941544.319159:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324605980 00000400:00000200:0.0:1773941544.319165:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941544.319171:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xda9 [1] + 0 00000800:00000001:0.0:1773941544.319176:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941544.319184:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941544.319186:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941544.319190:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941544.319195:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941544.319201:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d111f0a80 x1860112291486080/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941554 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941544.319213:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d111f0a80 x1860112291486080/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941554 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941544.319234:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941544.319240:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941544.319243:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941544.319270:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941544.319279:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941544.319282:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941544.319283:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941544.319285:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941544.319288:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941544.319289:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941544.319292:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941544.319298:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d111f0a80 x1860112291486080/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941554 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941544.319309:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d111f0a80 x1860112291486080/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941554 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941544.319317:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941544.319319:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941544.319322:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941544.319325:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941544.319326:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941544.319328:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941544.319331:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941544.319334:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=45 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941544.319338:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941544.319341:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941544.319342:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941544.319345:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941544.319347:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941544.319350:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941544.319370:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d111f0a80 x1860112291486080/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941554 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941544.319376:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941544.319378:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941544.319379:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941544.319380:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941544.319385:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941544.319386:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941544.319387:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941544.319388:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941544.319395:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941544.319396:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941544.319398:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:2.0:1773941544.319401:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941544 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941544.319403:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941544.319408:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:2.0:1773941544.319410:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941544.319411:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941544.319413:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941544.319414:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941544.319416:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:2.0:1773941544.319418:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941544.319420:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941544.319422:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941544.319424:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:2.0:1773941544.319426:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941544.319721:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941544.319725:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941544.319731:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d111f0a80 x1860112291486080/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941554 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941544.319739:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d111f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291486080:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941544.319741:0:14754:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:3.0:1773941544.319743:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1773941544.319750:0:14754:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00000001:3.0:1773941544.319754:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00100000:2.0:1773941544.319755:0:14754:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941544.319758:0:14754:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000040:3.0:1773941544.319761:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d111f0a80 x1860112291486080/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941554 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:01000000:2.0:1773941544.319761:0:14754:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:2.0:1773941544.319763:0:14754:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941544.319767:0:14754:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:3.0:1773941544.319771:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:2.0:1773941544.319772:0:14754:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941544.319774:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:2.0:1773941544.319775:0:14754:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000010:3.0:1773941544.319777:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a4c00. 00000001:00000001:2.0:1773941544.319777:0:14754:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:3.0:1773941544.319783:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:2.0:1773941544.319783:0:14754:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941544.319787:0:2400:0:(genops.c:1236:class_import_put()) Process entered 10000000:00000001:2.0:1773941544.319787:0:14754:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1773941544.319790:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941544.319790:0:14754:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000001:3.0:1773941544.319795:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00080000:2.0:1773941544.319796:0:14754:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 02000000:00000010:3.0:1773941544.319797:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d03522800. 00000100:00000001:2.0:1773941544.319800:0:14754:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 02000000:00000001:3.0:1773941544.319801:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1773941544.319803:0:14754:0:(recover.c:147:ptlrpc_resend()) Process entered 02000000:00000001:3.0:1773941544.319804:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1773941544.319806:0:14754:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941544.319807:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d111f0a80. 00000100:00080000:2.0:1773941544.319810:0:14754:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:3.0:1773941544.319812:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941544.319813:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941544.319814:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941544.319817:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941544.319819:0:14754:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:3.0:1773941544.319820:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 10000000:01000000:2.0:1773941544.319822:0:14754:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000100:00000001:3.0:1773941544.319823:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 10000000:00000040:2.0:1773941544.319825:0:14754:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00000001:3.0:1773941544.319826:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1773941544.319840:0:14754:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:2.0:1773941544.319842:0:14754:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941544.319843:0:14754:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:2.0:1773941544.319846:0:14754:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:2.0:1773941544.319848:0:14754:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941544.319850:0:14754:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941544.319851:0:14754:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941544.319853:0:14754:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941544.319854:0:14754:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:3.1:1773941544.323004:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d21ec1400. 00010000:00000010:3.1:1773941544.323010:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d00030600. 00000020:00000001:2.0:1773941544.955123:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941544.955129:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941544.955133:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941545.595063:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294935913, at 4294935936 00000100:00000010:1.0:1773941545.595079:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580e40. 00000100:00000040:0.0:1773941545.595088:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294935913, at 4294935936 00000100:00000010:0.0:1773941545.595103:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580b40. 00000020:00000001:0.0:1773941545.979174:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941545.979182:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941545.979188:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941546.235120:0:461:0:(pinger.c:229:ptlrpc_pinger_process_import()) c1c1ff21-5f68-4bf6-bdc4-258644b24465->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1773941546.235134:0:461:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1773941546.235149:0:461:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c2680. 00000020:00000040:1.0:1773941546.235157:0:461:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941546.235163:0:461:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941546.235167:0:461:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941546.235174:0:461:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941546.235195:0:461:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff988d02cfdd00. 00000100:00000001:1.0:1773941546.235203:0:461:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941546.235217:0:461:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging c1c1ff21-5f68-4bf6-bdc4-258644b24465->MGS req@ffff988d203c2680 x1860112291486208/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941546.235231:0:461:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941546.235237:0:461:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941546.235245:0:461:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000097d1012f] to pc [ptlrpcd_00_00+0] req@ffff988d203c2680 x1860112291486208/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941546.235292:0:461:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941546.235298:0:461:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (274) 00000100:00000001:1.0:1773941546.235308:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1773941546.235314:0:2402:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:1.0:1773941546.235333:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941546.235359:0:2402:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941546.235363:0:2402:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941546.235367:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941546.235370:0:2402:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941546.235373:0:2402:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1773941546.235378:0:2401:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1773941546.235381:0:2402:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c2680 x1860112291486208/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941546.235387:0:2401:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941546.235429:0:2402:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941546.235432:0:2402:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941546.235438:0:2402:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941546.235441:0:2402:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941546.235444:0:2402:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941546.235449:0:2402:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2402:1860112291486208:192.168.204.114@tcp:400:kworker.0 00000100:00000001:1.0:1773941546.235454:0:2402:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941546.235462:0:2402:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941546.235465:0:2402:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941546.235469:0:2402:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941546.235474:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff988d181fc200. 02000000:00000001:1.0:1773941546.235477:0:2402:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941546.235487:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941546.235492:0:2402:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c5d8. 00000400:00000010:1.0:1773941546.235501:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000100:00000200:1.0:1773941546.235511:0:2402:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112291486208, portal 25 00000100:00000001:1.0:1773941546.235514:0:2402:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941546.235516:0:2402:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631464576 : -113743078087040 : ffff988d203c2680) 00000100:00000040:1.0:1773941546.235523:0:2402:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c2680 x1860112291486208/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941562 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941546.235532:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941546.235536:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941546.235541:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de5d8. 00000100:00000200:1.0:1773941546.235545:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1860112291486208, offset 0 00000400:00000200:1.0:1773941546.235553:0:2402:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941546.235564:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941546.235574:0:2402:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941546.235581:0:2402:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941546.235585:0:2402:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941546.235590:0:2402:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941546.235594:0:2402:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941546.235603:0:2402:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[76, 75] 00000400:00000200:1.0:1773941546.235608:0:2402:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:1.0:1773941546.235614:0:2402:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [187:187:256:4294967295] 192.168.204.124@tcp LPNI seq info [77:77:8:4294967295] 00000400:00000200:1.0:1773941546.235620:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941546.235633:0:2402:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:1.0:1773941546.235653:0:2402:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:1.0:1773941546.235658:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfd300. 00000800:00000200:1.0:1773941546.235667:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:1.0:1773941546.235677:0:2402:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:1.0:1773941546.235683:0:2402:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfd300 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1773941546.235703:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941546.235708:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941546.235712:0:2402:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294937831 00000100:00000010:1.0:1773941546.235730:0:2402:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3300. 00000100:00000001:1.0:1773941546.235740:0:2402:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1773941546.235748:0:2402:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294937831 00000100:00000010:1.0:1773941546.235752:0:2402:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3360. 00000100:00000001:1.0:1773941546.235757:0:2402:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1773941546.235761:0:2402:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941546.235764:0:2402:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941546.235769:0:2402:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941546.235773:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941546.235836:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941546.235841:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfd300. 00000400:00000200:0.0:1773941546.235847:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941546.235855:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941546.235867:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de5d8 00000400:00000010:0.0:1773941546.235869:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de5d8. 00000100:00000001:0.0:1773941546.235874:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941546.235880:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c2680 x1860112291486208/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941562 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941546.235912:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941546.235917:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c2680 x1860112291486208/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941562 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941546.235925:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941546.235927:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941546.237021:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941546.237034:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941546.237037:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941546.237038:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941546.237044:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941546.237048:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941546.237054:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941546.237057:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941546.237061:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941546.237065:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324605a00 00000400:00000200:0.0:1773941546.237068:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941546.237072:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xdb1 [1] + 224 00000800:00000001:0.0:1773941546.237078:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941546.237084:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941546.237087:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941546.237090:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941546.237096:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941546.237102:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c2680 x1860112291486208/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941562 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941546.237111:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff988d203c2680 x1860112291486208/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941562 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941546.237125:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941546.237133:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941546.237135:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941546.237164:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941546.237168:0:2402:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941546.237174:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941546.237187:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941546.237190:0:2402:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:1.0:1773941546.237194:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:1.0:1773941546.237197:0:2402:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:1.0:1773941546.237200:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941546.237206:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c2680 x1860112291486208/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941562 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941546.237229:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c2680 x1860112291486208/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941562 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941546.237235:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941546.237236:0:2402:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941546.237238:0:2402:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941546.237241:0:2402:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941546.237244:0:2402:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941546.237246:0:2402:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941546.237248:0:2402:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941546.237254:0:2402:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=47 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941546.237257:0:2402:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941546.237259:0:2402:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1773941546.237261:0:2402:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941546.237264:0:2402:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941546.237266:0:2402:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941546.237270:0:2402:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941546.237274:0:2402:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941546.237277:0:2402:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941546.237279:0:2402:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941546.237280:0:2402:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941546.237282:0:2402:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941546.237285:0:2402:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d203c2680 x1860112291486208/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941562 ref 1 fl Rpc:RNQU/200/0 rc 0/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941546.237290:0:2402:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941546.237291:0:2402:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941546.237292:0:2402:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941546.237295:0:2402:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941546.237297:0:2402:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941546.237299:0:2402:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941546.237300:0:2402:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941546.237301:0:2402:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941546.237302:0:2402:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941546.237307:0:2402:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941546.237308:0:2402:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941546.237310:0:2402:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941546.237312:0:2402:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941546.237326:0:2402:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941546.237331:0:2402:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 267 00000400:00000200:1.0:1773941546.237335:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941546.237339:0:2402:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941546.237343:0:2402:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941546.237345:0:2402:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941546.237348:0:2402:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941546.237350:0:2402:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941546.237365:0:2402:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941546.237366:0:2402:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941546.237368:0:2402:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941546.237371:0:2402:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941546.237373:0:2402:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941546.237374:0:2402:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941546.237376:0:2402:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941546.237378:0:2402:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941546.237379:0:2402:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941546.237381:0:2402:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941546.237384:0:2402:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941546.237386:0:2402:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941546.237388:0:2402:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941546.237390:0:2402:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941546.237393:0:2402:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941546.237395:0:2402:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941546.237396:0:2402:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941546.237398:0:2402:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941546.237401:0:2402:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c0000. 00000020:00000040:1.0:1773941546.237402:0:2402:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941546.237409:0:2402:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941546.237411:0:2402:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941546.237413:0:2402:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941546.237415:0:2402:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941546.237417:0:2402:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941546.237431:0:2402:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d11189400. 00000100:00000001:1.0:1773941546.237434:0:2402:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941546.237439:0:2402:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c0000 x1860112291486336/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941546.237444:0:2402:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941546.237446:0:2402:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941546.237449:0:2402:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001ea6e639] to pc [ptlrpcd_rcv+-1] req@ffff988d203c0000 x1860112291486336/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941546.237465:0:2402:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941546.237467:0:2402:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941546.237469:0:2402:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:1.0:1773941546.237472:0:2402:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c2680 x1860112291486208/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941562 ref 1 fl Rpc:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941546.237478:0:2402:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941546.237480:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941546.237482:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941546.237486:0:2402:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c2680 x1860112291486208/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941562 ref 1 fl Interpret:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941546.237492:0:2402:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2402:1860112291486208:192.168.204.114@tcp:400:kworker.0 00000100:00000001:1.0:1773941546.237496:0:2402:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941546.237498:0:2402:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1773941546.237500:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:1.0:1773941546.237501:0:2402:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c2680 x1860112291486208/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941562 ref 1 fl Complete:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941546.237505:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941546.237508:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941546.237510:0:2402:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000040:3.0:1773941546.237516:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c0000 x1860112291486336/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1773941546.237517:0:2402:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941546.237522:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff988d181fc200. 00000100:00000001:3.0:1773941546.237528:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1773941546.237530:0:2402:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1773941546.237531:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941546.237534:0:2402:0:(genops.c:1236:class_import_put()) Process entered 02000000:00000001:3.0:1773941546.237535:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941546.237539:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000040:1.0:1773941546.237539:0:2402:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 02000000:00000001:3.0:1773941546.237544:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941546.237546:0:2402:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00100000:3.0:1773941546.237548:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291486336:192.168.204.114@tcp:250:ptlrpcd_00_01.0 00000100:00000001:3.0:1773941546.237552:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000010:1.0:1773941546.237552:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff988d02cfdd00. 02000000:00000001:3.0:1773941546.237555:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941546.237557:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941546.237559:0:2402:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941546.237563:0:2402:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941546.237566:0:2402:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c2680. 02000000:00000001:3.0:1773941546.237567:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941546.237572:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03522800. 00000100:00000001:1.0:1773941546.237574:0:2402:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 02000000:00000001:3.0:1773941546.237575:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941546.237589:0:2402:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941546.237593:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941546.237595:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1773941546.237597:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00000100:00000001:1.0:1773941546.237601:0:2402:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000400:00000010:3.0:1773941546.237602:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00000100:00000001:1.0:1773941546.237605:0:2402:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1773941546.237607:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291486336, portal 25 00000100:00000001:1.0:1773941546.237611:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941546.237612:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941546.237614:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631454720 : -113743078096896 : ffff988d203c0000) 00000100:00000001:1.0:1773941546.237617:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941546.237620:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c0000 x1860112291486336/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941556 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941546.237626:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941546.237628:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:3.0:1773941546.237630:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac088. 00000100:00000200:3.0:1773941546.237633:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291486336, offset 0 00000400:00000200:3.0:1773941546.237636:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941546.237640:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941546.237643:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941546.237645:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941546.237647:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941546.237649:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941546.237650:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941546.237652:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[76, 77] 00000400:00000200:3.0:1773941546.237655:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941546.237657:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [188:188:256:4294967295] 192.168.204.114@tcp LPNI seq info [78:78:8:4294967295] 00000400:00000200:3.0:1773941546.237660:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941546.237666:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:3.0:1773941546.237670:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:3.0:1773941546.237673:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74500. 00000800:00000200:3.0:1773941546.237676:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:3.0:1773941546.237682:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:3.0:1773941546.237688:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74500 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:3.0:1773941546.237700:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941546.237704:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:3.0:1773941546.237707:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294937833 00000100:00000001:3.0:1773941546.237715:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294937831 00000100:00000001:3.0:1773941546.237719:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294937833 00000100:00000001:3.0:1773941546.237725:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294937831 00000100:00000001:3.0:1773941546.237729:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941546.237732:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941546.237734:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941546.237737:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941546.237742:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941546.237745:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1773941546.237748:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941546.237750:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941546.237753:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941546.237755:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941546.237788:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941546.237795:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74500. 00000400:00000200:0.0:1773941546.237803:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941546.237807:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941546.237810:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac088 00000400:00000010:0.0:1773941546.237811:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac088. 00000100:00000001:0.0:1773941546.237814:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941546.237817:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c0000 x1860112291486336/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941556 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941546.237823:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941546.237825:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c0000 x1860112291486336/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941556 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941546.237830:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941546.237832:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941546.239352:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941546.239357:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941546.239359:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941546.239361:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941546.239364:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941546.239368:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941546.239372:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941546.239374:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941546.239377:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941546.239379:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324605a80 00000400:00000200:0.0:1773941546.239382:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941546.239385:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xdc1 [1] + 0 00000800:00000001:0.0:1773941546.239389:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941546.239393:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941546.239395:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941546.239397:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941546.239400:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941546.239404:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c0000 x1860112291486336/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941556 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941546.239412:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c0000 x1860112291486336/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941556 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941546.239431:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941546.239436:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941546.239438:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941546.239487:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941546.239490:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941546.239493:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941546.239495:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941546.239496:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941546.239500:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941546.239502:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941546.239504:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941546.239508:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c0000 x1860112291486336/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941556 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941546.239516:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c0000 x1860112291486336/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941556 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941546.239522:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941546.239523:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941546.239524:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941546.239526:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941546.239527:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941546.239534:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941546.239536:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941546.239539:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=47 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941546.239541:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941546.239543:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941546.239544:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941546.239547:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941546.239549:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941546.239551:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941546.239553:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c0000 x1860112291486336/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941556 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941546.239558:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941546.239560:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941546.239561:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941546.239563:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941546.239569:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941546.239571:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941546.239572:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941546.239573:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941546.239575:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941546.239576:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941546.239578:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941546.239581:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941546 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941546.239583:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941546.239588:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941546.239589:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941546.239591:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941546.239592:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941546.239594:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941546.239597:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941546.239598:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941546.239600:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941546.239602:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941546.239610:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941546.239612:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941546.239922:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941546.239926:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941546.239933:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c0000 x1860112291486336/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941556 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941546.239941:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291486336:192.168.204.114@tcp:250:ptlrpcd_00_01.0 00000100:00000001:3.0:1773941546.239943:0:14755:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00080000:3.0:1773941546.239952:0:14755:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00000001:2.0:1773941546.239954:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1773941546.239955:0:14755:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941546.239958:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1773941546.239982:0:14755:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941546.239985:0:14755:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:3.0:1773941546.239988:0:14755:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941546.239991:0:14755:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941546.239995:0:14755:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941546.239997:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c0000 x1860112291486336/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941556 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000001:00000001:3.0:1773941546.239998:0:14755:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941546.240001:0:14755:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941546.240003:0:14755:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941546.240007:0:14755:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941546.240011:0:14755:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941546.240012:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941546.240015:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00080000:3.0:1773941546.240016:0:14755:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 02000000:00000010:2.0:1773941546.240018:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03522800. 00000100:00000001:3.0:1773941546.240021:0:14755:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 02000000:00000001:2.0:1773941546.240022:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1773941546.240024:0:14755:0:(recover.c:147:ptlrpc_resend()) Process entered 00000020:00000001:2.0:1773941546.240025:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:3.0:1773941546.240027:0:14755:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941546.240028:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00080000:3.0:1773941546.240031:0:14755:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000020:00000001:2.0:1773941546.240032:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941546.240035:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d11189400. 00000100:00000001:3.0:1773941546.240036:0:14755:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941546.240039:0:14755:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 02000000:00000001:2.0:1773941546.240040:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 10000000:00000040:3.0:1773941546.240041:0:14755:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 02000000:00000001:2.0:1773941546.240049:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941546.240051:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c0000. 00000100:00000001:2.0:1773941546.240054:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00080000:3.0:1773941546.240056:0:14755:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:00000001:2.0:1773941546.240056:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941546.240058:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:3.0:1773941546.240059:0:14755:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941546.240062:0:14755:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941546.240063:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941546.240066:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:02000000:3.0:1773941546.240067:0:14755:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:2.0:1773941546.240069:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941546.240072:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941546.240073:0:14755:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941546.240076:0:14755:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941546.240078:0:14755:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941546.240080:0:14755:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941546.240082:0:14755:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941546.299047:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1773941546.299055:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941546.299060:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941546.299063:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941546.299065:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941546.299066:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1773941546.299069:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:0.0:1773941546.299071:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:0.0:1773941546.299073:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:0.0:1773941546.299076:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1773941546.299077:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:0.0:1773941546.299088:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10836300. 00000020:00000040:0.0:1773941546.299094:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941546.299111:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941546.299114:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941546.299120:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941546.299128:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065c5e00. 00000100:00000001:0.0:1773941546.299135:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941546.299140:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1773941546.299143:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1773941546.299151:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d21ec1800. 00010000:00000001:0.0:1773941546.299157:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1773941546.299160:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d00031600. 00000020:00000001:0.0:1773941546.299164:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:0.0:1773941546.299167:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d00031600 with handle 0x8e86767622e29ad to hash 00000020:00000001:0.0:1773941546.299168:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:0.0:1773941546.299170:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330090853888 : -113743618697728 : ffff988d00031600) 00010000:00000001:0.0:1773941546.299173:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330090853888 : -113743618697728 : ffff988d00031600) 00010000:00000001:0.0:1773941546.299177:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941546.299179:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941546.299183:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e29ad lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941546.299210:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e29ad lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941546.299222:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e29ad lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941546.299227:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1773941546.299229:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941546.299236:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb66c0. 00000100:00000001:0.0:1773941546.299238:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330139633344 : -113743569918272 : ffff988d02eb66c0) 00000100:00000001:0.0:1773941546.299241:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941546.299242:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367697664 : -113743341853952 : ffff988d10836300) 00000100:00000001:0.0:1773941546.299248:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941546.299253:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941546.299256:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941546.299258:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941546.299272:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10836300 x1860112291486464/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941546.299283:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941546.299287:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941546.299292:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941546.299295:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941546.299298:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941546.299303:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10836300 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291486464:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941546.299309:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941546.299327:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941546.299331:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941546.299334:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941546.299340:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d02ce4800. 02000000:00000001:0.0:1773941546.299343:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941546.299353:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941546.299377:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0014bbb0. 00000400:00000010:0.0:1773941546.299386:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833000. 00000100:00000200:0.0:1773941546.299395:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291486464, portal 25 00000100:00000001:0.0:1773941546.299397:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941546.299399:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367697664 : -113743341853952 : ffff988d10836300) 00000100:00000040:0.0:1773941546.299406:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10836300 x1860112291486464/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941562 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941546.299417:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941546.299420:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941546.299424:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d108332a8. 00000100:00000200:0.0:1773941546.299427:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291486464, offset 0 00000400:00000200:0.0:1773941546.299435:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941546.299445:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941546.299453:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941546.299458:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941546.299463:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941546.299467:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941546.299469:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941546.299471:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[78, 77] 00000400:00000200:0.0:1773941546.299474:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:0.0:1773941546.299478:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [189:189:256:4294967295] 192.168.204.124@tcp LPNI seq info [79:79:8:4294967295] 00000400:00000200:0.0:1773941546.299481:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941546.299490:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:0.0:1773941546.299495:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:0.0:1773941546.299500:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a20bd00. 00000800:00000200:0.0:1773941546.299507:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:0.0:1773941546.299514:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:0.0:1773941546.299519:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a20bd00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1773941546.299538:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941546.299542:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941546.299545:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294937895 00000100:00000001:0.0:1773941546.299558:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294937833 00000100:00000001:0.0:1773941546.299563:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294937895 00000100:00000001:0.0:1773941546.299567:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294937833 00000100:00000001:0.0:1773941546.299570:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941546.299572:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941546.299575:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941546.299577:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1773941546.299580:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb66c0 going to sleep for 11 seconds 00000100:00000001:0.0:1773941546.299583:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941546.299586:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941546.299664:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941546.299669:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a20bd00. 00000400:00000200:1.0:1773941546.299675:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941546.299681:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941546.299686:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d108332a8 00000400:00000010:1.0:1773941546.299689:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d108332a8. 00000100:00000001:1.0:1773941546.299692:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941546.299697:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10836300 x1860112291486464/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941562 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941546.299706:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941546.299709:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d10836300 x1860112291486464/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941562 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941546.299715:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941546.299716:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941546.300840:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941546.300858:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941546.300863:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941546.300865:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941546.300874:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941546.300883:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941546.300892:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941546.300897:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941546.300904:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941546.300910:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324605b00 00000400:00000200:0.0:1773941546.300916:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941546.300923:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xdc9 [1] + 224 00000800:00000001:0.0:1773941546.300930:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941546.300938:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941546.300947:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941546.300952:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941546.300959:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941546.300994:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10836300 x1860112291486464/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941562 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941546.301008:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d10836300 x1860112291486464/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941562 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941546.301053:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941546.301062:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941546.301066:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941546.301097:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941546.301104:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941546.301106:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941546.301109:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0014bbb0. 00000400:00000200:3.0:1773941546.301122:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833000 00000400:00000010:3.0:1773941546.301124:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833000. 00000100:00000001:3.0:1773941546.301128:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941546.301134:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10836300 x1860112291486464/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941562 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941546.301147:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10836300 x1860112291486464/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941562 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941546.301155:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941546.301157:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941546.301160:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941546.301163:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941546.301166:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941546.301168:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941546.301172:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941546.301180:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=47 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941546.301184:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=0 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941546.301187:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:3.0:1773941546.301192:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:3.0:1773941546.301195:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941546.301199:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:3.0:1773941546.301202:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:3.0:1773941546.301207:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1773941546.301210:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1773941546.301213:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:3.0:1773941546.301215:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941546.301216:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:3.0:1773941546.301218:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:3.0:1773941546.301221:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d10836300 x1860112291486464/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941562 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941546.301226:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941546.301227:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941546.301234:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941546.301237:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1773941546.301239:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:3.0:1773941546.301240:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941546.301241:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941546.301242:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941546.301243:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:3.0:1773941546.301244:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941546.301245:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941546.301247:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1773941546.301248:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1773941546.301251:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1773941546.301253:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 269 00000400:00000200:3.0:1773941546.301256:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941546.301260:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:3.0:1773941546.301263:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:3.0:1773941546.301265:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941546.301267:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:3.0:1773941546.301268:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:3.0:1773941546.301271:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941546.301273:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941546.301275:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941546.301277:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941546.301278:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941546.301280:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:3.0:1773941546.301282:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:3.0:1773941546.301283:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941546.301284:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:3.0:1773941546.301286:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:3.0:1773941546.301289:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:3.0:1773941546.301292:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:3.0:1773941546.301295:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941546.301297:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1773941546.301305:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941546.301308:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941546.301310:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:3.0:1773941546.301312:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941546.301342:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d111f0a80. 00000020:00000040:3.0:1773941546.301347:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:3.0:1773941546.301352:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1773941546.301353:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941546.301356:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941546.301359:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941546.301363:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941546.301368:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d03523000. 00000100:00000001:3.0:1773941546.301373:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941546.301379:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d111f0a80 x1860112291486592/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1773941546.301386:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941546.301389:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941546.301392:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b933c3a] to pc [ptlrpcd_rcv+-1] req@ffff988d111f0a80 x1860112291486592/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941546.301411:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941546.301413:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:3.0:1773941546.301414:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941546.301416:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941546.301419:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941546.301420:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1773941546.301422:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb66c0 going to sleep for 1 seconds 00000100:00000001:3.0:1773941546.301423:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941546.301425:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941546.301428:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10836300 x1860112291486464/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941562 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941546.301433:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941546.301436:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941546.301438:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941546.301444:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941546.301446:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:3.0:1773941546.301449:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10836300 x1860112291486464/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941562 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941546.301451:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941546.301454:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00100000:3.0:1773941546.301464:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10836300 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291486464:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000040:2.0:1773941546.301465:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d111f0a80 x1860112291486592/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941546.301470:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941546.301475:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941546.301480:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:2.0:1773941546.301480:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941546.301483:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941546.301483:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941546.301488:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:3.0:1773941546.301489:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10836300 x1860112291486464/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941562 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1773941546.301491:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941546.301497:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941546.301501:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941546.301502:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb66c0. 00000100:00000001:3.0:1773941546.301507:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00100000:2.0:1773941546.301507:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d111f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291486592:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941546.301511:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941546.301514:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00000001:3.0:1773941546.301516:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 02000000:00000001:2.0:1773941546.301518:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00000001:3.0:1773941546.301519:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 02000000:00000001:2.0:1773941546.301521:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941546.301522:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941546.301525:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00031600 refcount=4 02000000:00000001:2.0:1773941546.301525:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:3.0:1773941546.301529:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090853888 : -113743618697728 : ffff988d00031600) 00010000:00000001:3.0:1773941546.301531:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090853888 : -113743618697728 : ffff988d00031600) 02000000:00000010:2.0:1773941546.301531:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a5000. 02000000:00000001:2.0:1773941546.301534:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941546.301535:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e29ad lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941546.301541:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941546.301546:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00010000:00000001:3.0:1773941546.301548:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000010:2.0:1773941546.301551:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00010000:00010000:3.0:1773941546.301553:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e29ad lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941546.301559:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:3.0:1773941546.301561:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e29ad lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:2.0:1773941546.301563:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291486592, portal 25 00000100:00000001:2.0:1773941546.301567:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941546.301568:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330377898624 : -113743331652992 : ffff988d111f0a80) 00000100:00000040:2.0:1773941546.301576:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d111f0a80 x1860112291486592/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941556 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1773941546.301578:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941546.301581:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:3.0:1773941546.301585:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e29ad lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941546.301586:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:3.0:1773941546.301590:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941546.301592:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941546.301597:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000100:00000040:2.0:1773941546.301598:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00010000:3.0:1773941546.301601:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e29ad lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941546.301602:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516440. 00000100:00000200:2.0:1773941546.301609:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291486592, offset 0 00010000:00010000:3.0:1773941546.301610:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d00031600 already unused, calling callback (ffffffffc12b4c20) 10000000:00000001:3.0:1773941546.301614:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:2.0:1773941546.301615:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 10000000:00010000:3.0:1773941546.301617:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e29ad lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941546.301623:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:3.0:1773941546.301627:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:3.0:1773941546.301629:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1773941546.301632:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:2.0:1773941546.301632:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000020:00000040:3.0:1773941546.301635:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00031600 refcount=5 00000020:00000001:3.0:1773941546.301638:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090853888 : -113743618697728 : ffff988d00031600) 00000400:00000200:2.0:1773941546.301639:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:3.0:1773941546.301641:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090853888 : -113743618697728 : ffff988d00031600) 00000400:00000200:2.0:1773941546.301644:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941546.301647:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941546.301649:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941546.301653:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[78, 79] 00010000:00000001:3.0:1773941546.301654:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:3.0:1773941546.301658:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e29ad lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941546.301658:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:2.0:1773941546.301666:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [190:190:256:4294967295] 192.168.204.114@tcp LPNI seq info [80:80:8:4294967295] 10000000:00000001:3.0:1773941546.301668:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:3.0:1773941546.301672:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e29ad lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941546.301672:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 10000000:01000000:3.0:1773941546.301679:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:2.0:1773941546.301683:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 10000000:00000040:3.0:1773941546.301684:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000800:00000200:2.0:1773941546.301689:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 10000000:00000001:3.0:1773941546.301693:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1773941546.301695:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0db600. 00010000:00010000:3.0:1773941546.301696:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:3.0:1773941546.301699:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000800:00000200:2.0:1773941546.301703:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00010000:00000001:3.0:1773941546.301704:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:3.0:1773941546.301707:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:3.0:1773941546.301708:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941546.301710:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941546.301714:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000020:00000040:3.0:1773941546.301715:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d00031600 with handle 0x8e86767622e29ad from hash 00010000:00000001:3.0:1773941546.301719:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000800:00000200:2.0:1773941546.301719:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0db600 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:3.0:1773941546.301720:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941546.301723:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941546.301725:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:3.0:1773941546.301726:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:3.0:1773941546.301727:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:3.0:1773941546.301729:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941546.301730:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941546.301731:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941546.301732:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941546.301734:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e29ad lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941546.301735:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941546.301743:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:2.0:1773941546.301743:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:3.0:1773941546.301747:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941546.301748:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294937897 00010000:00000001:3.0:1773941546.301749:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:3.0:1773941546.301753:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:3.0:1773941546.301754:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941546.301755:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941546.301756:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941546.301757:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294937895 00010000:00010000:3.0:1773941546.301760:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e29ad lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941546.301763:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294937897 00010000:00000040:3.0:1773941546.301768:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d21ec1800 count: 0 00000100:00000001:2.0:1773941546.301770:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294937895 00000100:00000001:2.0:1773941546.301776:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941546.301778:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941546.301780:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941546.301782:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1773941546.301783:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941546.301785:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941546.301789:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941546.301798:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10836300 x1860112291486464/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941562 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941546.301798:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941546.301800:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941546.301803:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941546.301806:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941546.301809:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941546.301812:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941546.301818:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000800:00000200:1.0:1773941546.301822:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 02000000:00000001:3.0:1773941546.301823:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000800:00000010:1.0:1773941546.301830:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0db600. 02000000:00000010:3.0:1773941546.301832:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d02ce4800. 00000400:00000200:1.0:1773941546.301837:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:3.0:1773941546.301839:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941546.301843:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000400:00000200:1.0:1773941546.301845:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000020:00000040:3.0:1773941546.301846:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000400:00000200:1.0:1773941546.301850:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516440 00000020:00000001:3.0:1773941546.301853:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941546.301857:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065c5e00. 00000400:00000010:1.0:1773941546.301861:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516440. 00000100:00000001:1.0:1773941546.301867:0:2393:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:3.0:1773941546.301868:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000200:1.0:1773941546.301873:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d111f0a80 x1860112291486592/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941556 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941546.301882:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941546.301885:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d111f0a80 x1860112291486592/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941556 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941546.301890:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941546.301891:0:2393:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:3.0:1773941546.301915:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941546.301917:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10836300. 00000100:00000001:3.0:1773941546.301920:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941546.301921:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:3.0:1773941546.301922:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:3.0:1773941546.301924:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:3.0:1773941546.301926:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:3.0:1773941546.301929:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d03ec7540. 00000020:00000010:3.0:1773941546.301932:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d1a1c8c00. 00000020:00000010:3.0:1773941546.301936:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d1a1c8800. 00000040:00000010:3.0:1773941546.301939:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d1a1c9800. 00000020:00000010:3.0:1773941546.301942:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d08995000. 00000020:00000010:3.0:1773941546.301945:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d1a1c8200. 80000000:00000010:3.0:1773941546.301949:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d111f7c00. 00040000:00000010:3.0:1773941546.301952:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d1a1c8600. 20000000:00000010:3.0:1773941546.301955:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d1a1c8e00. 10000000:00000040:3.0:1773941546.301958:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:3.0:1773941546.301991:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:3.0:1773941546.301994:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d1a1c8800. 00000040:00000010:3.0:1773941546.301997:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d1a1c9800. 00000020:00000010:3.0:1773941546.301999:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d08995000. 00000020:00000010:3.0:1773941546.302003:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d1a1c8200. 80000000:00000010:3.0:1773941546.302006:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d111f7c00. 00040000:00000010:3.0:1773941546.302008:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d1a1c8600. 20000000:00000010:3.0:1773941546.302016:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d1a1c8e00. 00000020:00000010:3.0:1773941546.302019:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d1a1c8c00. 10000000:00000010:3.0:1773941546.302022:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d03ec7540. 10000000:01000000:3.0:1773941546.302025:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:3.0:1773941546.302028:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:3.0:1773941546.302029:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:3.0:1773941546.302031:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:3.0:1773941546.302032:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:3.0:1773941546.302034:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:3.0:1773941546.302035:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941546.303237:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941546.303246:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941546.303249:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941546.303252:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941546.303257:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941546.303262:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941546.303269:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941546.303273:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941546.303277:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941546.303280:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324605b80 00000400:00000200:0.0:1773941546.303284:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941546.303288:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xdd9 [1] + 0 00000800:00000001:0.0:1773941546.303293:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941546.303299:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941546.303302:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941546.303306:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941546.303311:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941546.303331:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d111f0a80 x1860112291486592/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941556 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941546.303343:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d111f0a80 x1860112291486592/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941556 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941546.303360:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941546.303365:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941546.303369:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941546.303422:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941546.303425:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941546.303428:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941546.303429:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941546.303431:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941546.303434:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941546.303436:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941546.303438:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941546.303442:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d111f0a80 x1860112291486592/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941556 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941546.303450:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d111f0a80 x1860112291486592/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941556 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941546.303455:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941546.303456:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941546.303457:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941546.303459:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941546.303460:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941546.303461:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941546.303462:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941546.303465:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=47 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941546.303467:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=0 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941546.303468:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941546.303470:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941546.303472:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941546.303473:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941546.303475:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941546.303478:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d111f0a80 x1860112291486592/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941556 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941546.303483:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941546.303484:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941546.303491:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941546.303493:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941546.303497:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941546.303499:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941546.303500:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941546.303501:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941546.303503:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941546.303504:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941546.303506:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:2.0:1773941546.303508:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941546 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941546.303510:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941546.303515:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:2.0:1773941546.303516:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941546.303517:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941546.303519:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941546.303520:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941546.303523:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:2.0:1773941546.303524:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941546.303526:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941546.303529:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941546.303531:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:2.0:1773941546.303533:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941546.303742:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941546.303745:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941546.303750:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d111f0a80 x1860112291486592/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941556 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941546.303757:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d111f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291486592:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941546.303761:0:14756:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:3.0:1773941546.303762:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941546.303765:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941546.303769:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d111f0a80 x1860112291486592/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941556 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1773941546.303771:0:14756:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:2.0:1773941546.303777:0:14756:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:3.0:1773941546.303780:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1773941546.303780:0:14756:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:3.0:1773941546.303783:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 10000000:01000000:2.0:1773941546.303784:0:14756:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000010:3.0:1773941546.303787:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a5000. 00000001:00000001:2.0:1773941546.303788:0:14756:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941546.303792:0:14756:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:3.0:1773941546.303793:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941546.303795:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:2.0:1773941546.303796:0:14756:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1773941546.303798:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000001:00000001:2.0:1773941546.303799:0:14756:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:3.0:1773941546.303802:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:2.0:1773941546.303802:0:14756:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000010:3.0:1773941546.303806:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d03523000. 00000001:00000001:2.0:1773941546.303806:0:14756:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941546.303811:0:14756:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941546.303812:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941546.303814:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941546.303815:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d111f0a80. 00000100:00000001:3.0:1773941546.303817:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941546.303820:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941546.303821:0:14756:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941546.303825:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1773941546.303827:0:14756:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941546.303829:0:14756:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941546.303831:0:14756:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:2.0:1773941546.303833:0:14756:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1773941546.303836:0:14756:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941546.303838:0:14756:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:3.0:1773941546.303839:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 10000000:01000000:2.0:1773941546.303839:0:14756:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:2.0:1773941546.303843:0:14756:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00000001:3.0:1773941546.303844:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941546.303848:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941546.303850:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1773941546.303856:0:14756:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:2.0:1773941546.303858:0:14756:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941546.303859:0:14756:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:2.0:1773941546.303862:0:14756:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:2.0:1773941546.303864:0:14756:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941546.303866:0:14756:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941546.303867:0:14756:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941546.303868:0:14756:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941546.303870:0:14756:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:3.1:1773941546.308003:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d21ec1800. 00010000:00000010:3.1:1773941546.308012:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d00031600. 00000020:00000001:2.0:1773941547.003128:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941547.003134:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941547.003138:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1773941547.579069:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294937897, at 4294937920 00000100:00000010:0.0:1773941547.579082:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3300. 00000100:00000040:1.0:1773941547.579131:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294937897, at 4294937920 00000100:00000010:1.0:1773941547.579145:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3360. 10000000:00000001:0.0:1773941547.835126:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1773941547.835169:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941547.835175:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941547.835179:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941547.835181:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941547.835183:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1773941547.835185:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:0.0:1773941547.835188:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:0.0:1773941547.835192:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:0.0:1773941547.835195:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1773941547.835198:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:0.0:1773941547.835205:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10836a00. 00000020:00000040:0.0:1773941547.835210:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941547.835215:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941547.835218:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941547.835223:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941547.835233:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065c4a00. 00000100:00000001:0.0:1773941547.835240:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941547.835244:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1773941547.835247:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1773941547.835254:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d21ec1800. 00010000:00000001:0.0:1773941547.835259:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1773941547.835262:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d00031800. 00000020:00000001:0.0:1773941547.835265:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:0.0:1773941547.835267:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d00031800 with handle 0x8e86767622e29b4 to hash 00000020:00000001:0.0:1773941547.835269:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:0.0:1773941547.835270:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330090854400 : -113743618697216 : ffff988d00031800) 00010000:00000001:0.0:1773941547.835272:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330090854400 : -113743618697216 : ffff988d00031800) 00010000:00000001:0.0:1773941547.835275:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941547.835279:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941547.835282:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00031800/0x8e86767622e29b4 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941547.835308:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d00031800/0x8e86767622e29b4 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941547.835327:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d00031800/0x8e86767622e29b4 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941547.835333:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1773941547.835334:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941547.835340:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb6840. 00000100:00000001:0.0:1773941547.835341:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330139633728 : -113743569917888 : ffff988d02eb6840) 00000100:00000001:0.0:1773941547.835344:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941547.835345:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367699456 : -113743341852160 : ffff988d10836a00) 00000100:00000001:0.0:1773941547.835349:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941547.835352:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941547.835354:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941547.835356:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941547.835364:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10836a00 x1860112291486720/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941547.835371:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941547.835373:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941547.835375:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941547.835378:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941547.835379:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941547.835382:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10836a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291486720:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941547.835385:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941547.835390:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941547.835391:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941547.835393:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941547.835396:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d02ce6800. 02000000:00000001:0.0:1773941547.835398:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941547.835404:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941547.835407:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0014b420. 00000400:00000010:0.0:1773941547.835411:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833b28. 00000100:00000200:0.0:1773941547.835417:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291486720, portal 25 00000100:00000001:0.0:1773941547.835418:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941547.835424:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367699456 : -113743341852160 : ffff988d10836a00) 00000100:00000040:0.0:1773941547.835429:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10836a00 x1860112291486720/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941563 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941547.835435:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941547.835437:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941547.835439:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833220. 00000100:00000200:0.0:1773941547.835441:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291486720, offset 0 00000400:00000200:0.0:1773941547.835446:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941547.835453:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941547.835459:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941547.835463:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941547.835466:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941547.835471:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941547.835473:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941547.835475:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[80, 79] 00000400:00000200:0.0:1773941547.835478:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:0.0:1773941547.835482:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [191:191:256:4294967295] 192.168.204.124@tcp LPNI seq info [81:81:8:4294967295] 00000400:00000200:0.0:1773941547.835485:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941547.835492:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:0.0:1773941547.835497:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:0.0:1773941547.835501:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a20bd00. 00000800:00000200:0.0:1773941547.835507:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:0.0:1773941547.835513:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:0.0:1773941547.835516:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a20bd00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1773941547.835529:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941547.835532:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941547.835534:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294939431 00000100:00000010:0.0:1773941547.835537:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580b40. 00000100:00000001:0.0:1773941547.835543:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1773941547.835549:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294939431 00000100:00000010:0.0:1773941547.835555:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580ea0. 00000100:00000001:0.0:1773941547.835558:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1773941547.835561:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941547.835563:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941547.835564:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941547.835566:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1773941547.835568:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6840 going to sleep for 11 seconds 00000100:00000001:0.0:1773941547.835570:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941547.835573:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941547.835638:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941547.835643:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a20bd00. 00000400:00000200:1.0:1773941547.835647:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941547.835652:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941547.835657:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833220 00000400:00000010:1.0:1773941547.835659:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833220. 00000100:00000001:1.0:1773941547.835662:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941547.835666:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10836a00 x1860112291486720/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941563 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941547.835675:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941547.835678:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d10836a00 x1860112291486720/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941563 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941547.835683:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941547.835684:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941547.836789:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941547.836802:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941547.836804:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941547.836806:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941547.836810:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941547.836814:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941547.836818:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941547.836821:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941547.836825:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941547.836828:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324605c00 00000400:00000200:0.0:1773941547.836831:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941547.836834:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xde1 [1] + 224 00000800:00000001:0.0:1773941547.836840:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941547.836849:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941547.836851:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941547.836855:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941547.836860:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941547.836865:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10836a00 x1860112291486720/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941563 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941547.836879:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d10836a00 x1860112291486720/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941563 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941547.836906:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941547.836914:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941547.836927:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941547.836945:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941547.836958:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941547.836991:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941547.836995:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0014b420. 00000400:00000200:3.0:1773941547.837003:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833b28 00000400:00000010:3.0:1773941547.837005:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833b28. 00000100:00000001:3.0:1773941547.837010:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941547.837021:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10836a00 x1860112291486720/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941563 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941547.837036:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10836a00 x1860112291486720/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941563 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941547.837044:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941547.837047:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941547.837050:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941547.837054:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941547.837057:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941547.837060:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941547.837063:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941547.837072:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=48 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941547.837076:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941547.837080:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:3.0:1773941547.837084:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:3.0:1773941547.837088:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941547.837091:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:3.0:1773941547.837095:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:3.0:1773941547.837101:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1773941547.837106:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1773941547.837111:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:3.0:1773941547.837114:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941547.837128:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:3.0:1773941547.837131:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:3.0:1773941547.837136:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d10836a00 x1860112291486720/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941563 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941547.837145:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941547.837147:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941547.837150:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941547.837154:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1773941547.837157:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:3.0:1773941547.837160:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941547.837162:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941547.837164:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941547.837165:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:3.0:1773941547.837167:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941547.837169:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941547.837172:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1773941547.837174:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1773941547.837178:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1773941547.837181:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 269 00000400:00000200:3.0:1773941547.837189:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941547.837195:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:3.0:1773941547.837200:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:3.0:1773941547.837204:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941547.837207:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:3.0:1773941547.837210:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:3.0:1773941547.837214:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941547.837216:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941547.837219:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941547.837222:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941547.837224:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941547.837227:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:3.0:1773941547.837231:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:3.0:1773941547.837234:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941547.837236:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:3.0:1773941547.837246:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:3.0:1773941547.837250:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:3.0:1773941547.837254:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:3.0:1773941547.837258:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941547.837260:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1773941547.837264:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941547.837279:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941547.837283:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:3.0:1773941547.837298:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941547.837305:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d111f0a80. 00000020:00000040:3.0:1773941547.837309:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:3.0:1773941547.837313:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1773941547.837315:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941547.837319:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941547.837322:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941547.837325:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941547.837332:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d03523000. 00000100:00000001:3.0:1773941547.837338:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941547.837346:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d111f0a80 x1860112291486848/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1773941547.837356:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941547.837361:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941547.837367:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b933c3a] to pc [ptlrpcd_rcv+-1] req@ffff988d111f0a80 x1860112291486848/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941547.837389:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941547.837393:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:3.0:1773941547.837395:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941547.837398:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941547.837402:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941547.837404:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1773941547.837407:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6840 going to sleep for 1 seconds 00000100:00000001:3.0:1773941547.837410:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941547.837413:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941547.837416:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941547.837421:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941547.837424:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941547.837428:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10836a00 x1860112291486720/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941563 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941547.837433:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d111f0a80 x1860112291486848/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941547.837451:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941547.837453:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941547.837455:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941547.837458:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1773941547.837460:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941547.837463:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941547.837465:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1773941547.837467:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941547.837470:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941547.837471:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d111f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291486848:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941547.837477:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941547.837480:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000040:3.0:1773941547.837481:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10836a00 x1860112291486720/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941563 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1773941547.837490:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941547.837494:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:3.0:1773941547.837499:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10836a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291486720:192.168.204.114@tcp:101:ll_cfg_requeue.0 02000000:00000010:2.0:1773941547.837500:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a7c00. 02000000:00000001:2.0:1773941547.837505:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941547.837508:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:2.0:1773941547.837513:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:3.0:1773941547.837515:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000010:2.0:1773941547.837518:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000100:00000001:3.0:1773941547.837521:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000400:00000010:2.0:1773941547.837524:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000001:3.0:1773941547.837525:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000200:2.0:1773941547.837530:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291486848, portal 25 00000100:00000040:3.0:1773941547.837532:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10836a00 x1860112291486720/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941563 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941547.837534:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941547.837537:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330377898624 : -113743331652992 : ffff988d111f0a80) 00000100:00000040:2.0:1773941547.837547:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d111f0a80 x1860112291486848/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941557 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941547.837549:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941547.837556:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb6840. 00000100:00000001:2.0:1773941547.837559:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1773941547.837563:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000040:2.0:1773941547.837563:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000100:00000001:3.0:1773941547.837568:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000010:2.0:1773941547.837569:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516550. 00010000:00000001:3.0:1773941547.837575:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000200:2.0:1773941547.837576:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291486848, offset 0 00010000:00000001:3.0:1773941547.837582:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:2.0:1773941547.837583:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000020:00000001:3.0:1773941547.837587:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941547.837592:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00031800 refcount=4 00000400:00000200:2.0:1773941547.837592:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000020:00000001:3.0:1773941547.837598:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090854400 : -113743618697216 : ffff988d00031800) 00000400:00000200:2.0:1773941547.837601:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:3.0:1773941547.837603:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090854400 : -113743618697216 : ffff988d00031800) 00000400:00000200:2.0:1773941547.837606:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941547.837611:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00010000:3.0:1773941547.837613:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d00031800/0x8e86767622e29b4 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941547.837615:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941547.837621:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941547.837627:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[80, 81] 00010000:00000001:3.0:1773941547.837633:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000200:2.0:1773941547.837634:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:2.0:1773941547.837640:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [192:192:256:4294967295] 192.168.204.114@tcp LPNI seq info [82:82:8:4294967295] 00010000:00010000:3.0:1773941547.837642:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d00031800/0x8e86767622e29b4 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941547.837644:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941547.837656:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:2.0:1773941547.837666:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00010000:00000001:3.0:1773941547.837668:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000800:00000010:2.0:1773941547.837672:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0db600. 00010000:00010000:3.0:1773941547.837674:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00031800/0x8e86767622e29b4 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941547.837681:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00010000:00000001:3.0:1773941547.837690:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000200:2.0:1773941547.837694:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00010000:00000001:3.0:1773941547.837695:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:3.0:1773941547.837698:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d00031800/0x8e86767622e29b4 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941547.837707:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0db600 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:3.0:1773941547.837711:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941547.837716:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941547.837720:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000100:00000001:2.0:1773941547.837723:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941547.837724:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d00031800/0x8e86767622e29b4 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941547.837732:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941547.837737:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294939433 00010000:00010000:3.0:1773941547.837744:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d00031800 already unused, calling callback (ffffffffc12b4c20) 00000100:00000001:2.0:1773941547.837746:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294939431 10000000:00000001:3.0:1773941547.837748:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000001:2.0:1773941547.837751:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294939433 10000000:00010000:3.0:1773941547.837753:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d00031800/0x8e86767622e29b4 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941547.837756:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294939431 00000100:00000001:2.0:1773941547.837763:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941547.837768:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941547.837771:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000100:00000001:2.0:1773941547.837772:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941547.837775:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1773941547.837779:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1773941547.837779:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1773941547.837784:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00031800 refcount=5 00000020:00000001:3.0:1773941547.837788:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090854400 : -113743618697216 : ffff988d00031800) 00000100:00000001:2.0:1773941547.837791:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:3.0:1773941547.837792:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090854400 : -113743618697216 : ffff988d00031800) 00000100:00000001:2.0:1773941547.837796:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00010000:00000001:3.0:1773941547.837799:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000100:00000001:2.0:1773941547.837802:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00010000:3.0:1773941547.837804:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d00031800/0x8e86767622e29b4 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941547.837806:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941547.837810:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941547.837815:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941547.837817:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:3.0:1773941547.837828:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d00031800/0x8e86767622e29b4 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:1.0:1773941547.837829:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941547.837836:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0db600. 00000400:00000200:1.0:1773941547.837845:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 10000000:01000000:3.0:1773941547.837849:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:1.0:1773941547.837854:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 10000000:00000040:3.0:1773941547.837861:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000400:00000200:1.0:1773941547.837863:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516550 10000000:00000001:3.0:1773941547.837866:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1773941547.837867:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516550. 00000100:00000001:1.0:1773941547.837875:0:2393:0:(events.c:34:request_out_callback()) Process entered 00010000:00010000:3.0:1773941547.837877:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:3.0:1773941547.837880:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:3.0:1773941547.837884:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000100:00000200:1.0:1773941547.837888:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d111f0a80 x1860112291486848/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941557 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941547.837895:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941547.837898:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d111f0a80 x1860112291486848/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941557 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941547.837903:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941547.837904:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:3.0:1773941547.837934:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:3.0:1773941547.837937:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941547.837939:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1773941547.837942:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d00031800 with handle 0x8e86767622e29b4 from hash 00010000:00000001:3.0:1773941547.837946:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:3.0:1773941547.837948:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941547.837949:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941547.837951:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:3.0:1773941547.837953:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:3.0:1773941547.837955:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:3.0:1773941547.837958:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941547.837980:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941547.837982:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941547.837985:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941547.837988:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d00031800/0x8e86767622e29b4 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941547.837995:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941547.837996:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941547.837998:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:3.0:1773941547.838000:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:3.0:1773941547.838002:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941547.838003:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941547.838005:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:3.0:1773941547.838007:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d00031800/0x8e86767622e29b4 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:3.0:1773941547.838016:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d21ec1800 count: 0 00010000:00000001:3.0:1773941547.838048:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941547.838051:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941547.838055:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941547.838062:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10836a00 x1860112291486720/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941563 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941547.838071:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941547.838073:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941547.838077:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d02ce6800. 02000000:00000001:3.0:1773941547.838081:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941547.838093:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941547.838095:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941547.838098:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941547.838101:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065c4a00. 02000000:00000001:3.0:1773941547.838107:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941547.838109:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941547.838111:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10836a00. 00000100:00000001:3.0:1773941547.838115:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941547.838117:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:3.0:1773941547.838120:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:3.0:1773941547.838123:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:3.0:1773941547.838126:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:3.0:1773941547.838130:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d03ec7540. 00000020:00000010:3.0:1773941547.838136:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d1a1c8c00. 00000020:00000010:3.0:1773941547.838141:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d1a1c8e00. 00000040:00000010:3.0:1773941547.838146:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d1a1c8600. 00000020:00000010:3.0:1773941547.838152:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d08995000. 00000020:00000010:3.0:1773941547.838157:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d1a1c8200. 80000000:00000010:3.0:1773941547.838162:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d111f7c00. 00040000:00000010:3.0:1773941547.838168:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d1a1c9800. 20000000:00000010:3.0:1773941547.838173:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d1a1c8800. 10000000:00000040:3.0:1773941547.838179:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:3.0:1773941547.838182:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:3.0:1773941547.838188:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d1a1c8e00. 00000040:00000010:3.0:1773941547.838193:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d1a1c8600. 00000020:00000010:3.0:1773941547.838203:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d08995000. 00000020:00000010:3.0:1773941547.838208:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d1a1c8200. 80000000:00000010:3.0:1773941547.838211:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d111f7c00. 00040000:00000010:3.0:1773941547.838215:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d1a1c9800. 20000000:00000010:3.0:1773941547.838219:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d1a1c8800. 00000020:00000010:3.0:1773941547.838222:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d1a1c8c00. 10000000:00000010:3.0:1773941547.838225:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d03ec7540. 10000000:01000000:3.0:1773941547.838230:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:3.0:1773941547.838234:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:3.0:1773941547.838237:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:3.0:1773941547.838239:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:3.0:1773941547.838242:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:3.0:1773941547.838244:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:3.0:1773941547.838246:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941547.839393:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941547.839401:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941547.839403:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941547.839405:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941547.839409:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941547.839413:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941547.839418:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941547.839420:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941547.839423:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941547.839425:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324605c80 00000400:00000200:0.0:1773941547.839430:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941547.839433:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xdf1 [1] + 0 00000800:00000001:0.0:1773941547.839439:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941547.839445:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941547.839446:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941547.839449:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941547.839452:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941547.839456:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d111f0a80 x1860112291486848/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941557 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941547.839463:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d111f0a80 x1860112291486848/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941557 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941547.839479:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941547.839483:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941547.839485:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941547.839510:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941547.839513:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941547.839516:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941547.839518:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941547.839520:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941547.839524:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941547.839526:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941547.839528:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941547.839534:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d111f0a80 x1860112291486848/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941557 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941547.839542:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d111f0a80 x1860112291486848/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941557 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941547.839548:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941547.839549:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941547.839550:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941547.839552:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941547.839553:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941547.839560:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941547.839562:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941547.839565:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=48 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941547.839568:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941547.839570:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941547.839571:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941547.839573:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941547.839575:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941547.839577:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941547.839580:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d111f0a80 x1860112291486848/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941557 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941547.839585:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941547.839587:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941547.839588:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941547.839590:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941547.839594:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941547.839596:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941547.839597:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941547.839598:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941547.839600:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941547.839601:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941547.839603:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:2.0:1773941547.839606:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941547 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941547.839607:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941547.839613:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:2.0:1773941547.839614:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941547.839616:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941547.839617:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941547.839619:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941547.839621:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:2.0:1773941547.839623:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941547.839625:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941547.839628:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941547.839634:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:2.0:1773941547.839636:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941547.839842:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941547.839847:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941547.839855:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d111f0a80 x1860112291486848/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941557 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941547.839861:0:14757:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:3.0:1773941547.839869:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d111f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291486848:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00080000:2.0:1773941547.839873:0:14757:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:2.0:1773941547.839879:0:14757:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:3.0:1773941547.839880:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941547.839882:0:14757:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941547.839885:0:14757:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000001:3.0:1773941547.839886:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000001:00000001:2.0:1773941547.839888:0:14757:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941547.839893:0:14757:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000040:3.0:1773941547.839894:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d111f0a80 x1860112291486848/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941557 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000001:00000001:2.0:1773941547.839897:0:14757:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:2.0:1773941547.839901:0:14757:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941547.839904:0:14757:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941547.839906:0:14757:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941547.839910:0:14757:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941547.839914:0:14757:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941547.839916:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00080000:2.0:1773941547.839919:0:14757:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941547.839922:0:14757:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941547.839924:0:14757:0:(recover.c:147:ptlrpc_resend()) Process entered 02000000:00000001:3.0:1773941547.839928:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1773941547.839931:0:14757:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941547.839933:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a7c00. 00000100:00080000:2.0:1773941547.839935:0:14757:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941547.839940:0:14757:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:3.0:1773941547.839941:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 10000000:01000000:2.0:1773941547.839942:0:14757:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000020:00000001:3.0:1773941547.839946:0:2400:0:(genops.c:1236:class_import_put()) Process entered 10000000:00000040:2.0:1773941547.839946:0:14757:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000020:00000040:3.0:1773941547.839951:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941547.839955:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941547.839957:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d03523000. 02000000:00000001:3.0:1773941547.839989:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00080000:2.0:1773941547.839990:0:14757:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:2.0:1773941547.839992:0:14757:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941547.839994:0:14757:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:2.0:1773941547.839997:0:14757:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 02000000:00000001:3.0:1773941547.840000:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941547.840006:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d111f0a80. 00000100:00000001:2.0:1773941547.840006:0:14757:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941547.840011:0:14757:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:3.0:1773941547.840014:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000020:00000040:2.0:1773941547.840015:0:14757:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941547.840020:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1773941547.840021:0:14757:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941547.840024:0:14757:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941547.840027:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941547.840035:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941547.840038:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941547.840042:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941547.840045:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:3.1:1773941547.843992:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d21ec1800. 00010000:00000010:3.1:1773941547.843999:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d00031800. 00000020:00000001:0.0:1773941548.027163:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941548.027171:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941548.027176:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1773941549.051157:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941549.051164:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941549.051168:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941549.115059:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294939433, at 4294939456 00000100:00000010:1.0:1773941549.115074:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580ea0. 00000100:00000040:0.0:1773941549.115074:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294939433, at 4294939456 00000100:00000010:0.0:1773941549.115087:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580b40. 10000000:00000001:0.0:1773941549.819089:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1773941549.819095:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941549.819098:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941549.819112:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941549.819113:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941549.819115:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1773941549.819117:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:0.0:1773941549.819119:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:0.0:1773941549.819121:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:0.0:1773941549.819123:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1773941549.819125:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:0.0:1773941549.819131:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10835180. 00000020:00000040:0.0:1773941549.819135:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941549.819140:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941549.819143:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941549.819147:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941549.819153:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065c5a00. 00000100:00000001:0.0:1773941549.819159:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941549.819162:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1773941549.819165:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1773941549.819171:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d21ec1400. 00010000:00000001:0.0:1773941549.819176:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1773941549.819179:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d00030c00. 00000020:00000001:0.0:1773941549.819182:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:0.0:1773941549.819184:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d00030c00 with handle 0x8e86767622e29bb to hash 00000020:00000001:0.0:1773941549.819186:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:0.0:1773941549.819187:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330090851328 : -113743618700288 : ffff988d00030c00) 00010000:00000001:0.0:1773941549.819189:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330090851328 : -113743618700288 : ffff988d00030c00) 00010000:00000001:0.0:1773941549.819192:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941549.819195:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941549.819198:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e29bb lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941549.819207:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e29bb lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941549.819216:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e29bb lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941549.819226:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1773941549.819228:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941549.819234:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb6600. 00000100:00000001:0.0:1773941549.819236:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330139633152 : -113743569918464 : ffff988d02eb6600) 00000100:00000001:0.0:1773941549.819238:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941549.819240:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367693184 : -113743341858432 : ffff988d10835180) 00000100:00000001:0.0:1773941549.819243:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941549.819247:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941549.819267:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941549.819269:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941549.819278:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10835180 x1860112291486976/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941549.819285:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941549.819287:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941549.819290:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941549.819292:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941549.819294:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941549.819297:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10835180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291486976:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941549.819301:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941549.819306:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941549.819308:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941549.819310:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941549.819314:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d02ce6000. 02000000:00000001:0.0:1773941549.819315:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941549.819322:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941549.819325:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0014b790. 00000400:00000010:0.0:1773941549.819330:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833110. 00000100:00000200:0.0:1773941549.819335:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291486976, portal 25 00000100:00000001:0.0:1773941549.819337:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941549.819338:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367693184 : -113743341858432 : ffff988d10835180) 00000100:00000040:0.0:1773941549.819342:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10835180 x1860112291486976/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941565 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941549.819355:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941549.819357:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941549.819359:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d108333b8. 00000100:00000200:0.0:1773941549.819361:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291486976, offset 0 00000400:00000200:0.0:1773941549.819366:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941549.819373:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941549.819378:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941549.819382:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941549.819385:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941549.819389:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941549.819391:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941549.819393:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[82, 81] 00000400:00000200:0.0:1773941549.819396:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:0.0:1773941549.819399:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [193:193:256:4294967295] 192.168.204.124@tcp LPNI seq info [83:83:8:4294967295] 00000400:00000200:0.0:1773941549.819402:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941549.819409:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:0.0:1773941549.819414:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:0.0:1773941549.819418:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a20bd00. 00000800:00000200:0.0:1773941549.819423:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:0.0:1773941549.819429:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:0.0:1773941549.819433:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a20bd00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1773941549.819447:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941549.819450:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941549.819452:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294941415 00000100:00000010:0.0:1773941549.819455:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580b40. 00000100:00000001:0.0:1773941549.819460:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1773941549.819465:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294941415 00000100:00000010:0.0:1773941549.819467:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580300. 00000100:00000001:0.0:1773941549.819469:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1773941549.819477:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941549.819479:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941549.819481:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941549.819482:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1773941549.819485:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6600 going to sleep for 11 seconds 00000100:00000001:0.0:1773941549.819487:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941549.819489:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941549.819573:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941549.819578:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a20bd00. 00000400:00000200:1.0:1773941549.819583:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941549.819590:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941549.819595:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d108333b8 00000400:00000010:1.0:1773941549.819596:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d108333b8. 00000100:00000001:1.0:1773941549.819600:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941549.819604:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10835180 x1860112291486976/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941565 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941549.819613:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941549.819615:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d10835180 x1860112291486976/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941565 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941549.819621:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941549.819623:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941549.820728:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941549.820741:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941549.820743:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941549.820746:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941549.820751:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941549.820755:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941549.820760:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941549.820763:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941549.820768:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941549.820771:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324605d00 00000400:00000200:0.0:1773941549.820776:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941549.820780:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xdf9 [1] + 224 00000800:00000001:0.0:1773941549.820786:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941549.820792:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941549.820794:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941549.820797:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941549.820800:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941549.820804:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10835180 x1860112291486976/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941565 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941549.820811:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d10835180 x1860112291486976/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941565 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941549.820829:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941549.820835:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941549.820838:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941549.820876:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941549.820888:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941549.820890:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941549.820894:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0014b790. 00000400:00000200:3.0:1773941549.820900:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833110 00000400:00000010:3.0:1773941549.820902:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833110. 00000100:00000001:3.0:1773941549.820905:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941549.820914:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10835180 x1860112291486976/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941565 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941549.820923:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10835180 x1860112291486976/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941565 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941549.820929:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941549.820930:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941549.820932:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941549.820935:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941549.820938:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941549.820940:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941549.820942:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941549.820948:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=50 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941549.820950:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941549.820953:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:3.0:1773941549.820957:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:3.0:1773941549.821005:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941549.821008:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:3.0:1773941549.821011:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:3.0:1773941549.821015:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1773941549.821032:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1773941549.821035:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:3.0:1773941549.821037:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941549.821038:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:3.0:1773941549.821040:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:3.0:1773941549.821044:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d10835180 x1860112291486976/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941565 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941549.821051:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941549.821053:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941549.821055:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941549.821059:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1773941549.821061:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:3.0:1773941549.821063:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941549.821066:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941549.821068:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941549.821070:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:3.0:1773941549.821072:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941549.821074:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941549.821077:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1773941549.821079:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1773941549.821082:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1773941549.821085:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 270 00000400:00000200:3.0:1773941549.821090:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941549.821095:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:3.0:1773941549.821098:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:3.0:1773941549.821100:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941549.821102:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:3.0:1773941549.821104:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:3.0:1773941549.821107:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941549.821109:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941549.821111:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941549.821114:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941549.821116:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941549.821121:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:3.0:1773941549.821123:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:3.0:1773941549.821125:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941549.821126:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:3.0:1773941549.821128:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:3.0:1773941549.821130:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:3.0:1773941549.821133:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:3.0:1773941549.821134:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941549.821136:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1773941549.821139:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941549.821141:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941549.821143:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:3.0:1773941549.821145:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941549.821149:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d111f0a80. 00000020:00000040:3.0:1773941549.821151:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:3.0:1773941549.821154:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1773941549.821156:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941549.821158:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941549.821160:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941549.821163:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941549.821168:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d03523000. 00000100:00000001:3.0:1773941549.821172:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941549.821177:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d111f0a80 x1860112291487104/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1773941549.821184:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941549.821187:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941549.821191:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b933c3a] to pc [ptlrpcd_rcv+-1] req@ffff988d111f0a80 x1860112291487104/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941549.821210:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941549.821213:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:3.0:1773941549.821214:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941549.821217:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941549.821223:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941549.821224:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1773941549.821226:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6600 going to sleep for 1 seconds 00000100:00000001:3.0:1773941549.821228:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941549.821229:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941549.821233:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10835180 x1860112291486976/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941565 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941549.821239:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941549.821241:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941549.821243:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941549.821245:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941549.821259:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10835180 x1860112291486976/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941565 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941549.821267:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10835180 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291486976:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941549.821273:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941549.821276:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941549.821279:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:3.0:1773941549.821282:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941549.821288:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10835180 x1860112291486976/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941565 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941549.821297:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941549.821300:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:3.0:1773941549.821301:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb6600. 00000100:00000001:3.0:1773941549.821305:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:3.0:1773941549.821307:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:3.0:1773941549.821311:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000001:2.0:1773941549.821314:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00010000:00000001:3.0:1773941549.821316:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000001:2.0:1773941549.821318:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1773941549.821319:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941549.821323:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030c00 refcount=4 00000020:00000001:3.0:1773941549.821325:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090851328 : -113743618700288 : ffff988d00030c00) 00010000:00000001:3.0:1773941549.821328:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090851328 : -113743618700288 : ffff988d00030c00) 00000100:00000040:2.0:1773941549.821330:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d111f0a80 x1860112291487104/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:3.0:1773941549.821333:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e29bb lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941549.821349:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:3.0:1773941549.821353:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941549.821353:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941549.821361:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e29bb lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941549.821361:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941549.821365:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941549.821370:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941549.821374:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00100000:2.0:1773941549.821375:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d111f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291487104:192.168.204.114@tcp:250:ll_cfg_requeue.0 00010000:00010000:3.0:1773941549.821378:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e29bb lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941549.821380:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941549.821385:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00000001:3.0:1773941549.821386:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 02000000:00000001:2.0:1773941549.821388:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941549.821389:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 02000000:00000001:2.0:1773941549.821392:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00010000:3.0:1773941549.821393:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e29bb lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000010:2.0:1773941549.821399:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a7c00. 00010000:00000001:3.0:1773941549.821400:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941549.821403:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941549.821404:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941549.821408:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:3.0:1773941549.821412:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e29bb lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941549.821412:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941549.821416:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00010000:00010000:3.0:1773941549.821419:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d00030c00 already unused, calling callback (ffffffffc12b4c20) 00000400:00000010:2.0:1773941549.821422:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 10000000:00000001:3.0:1773941549.821423:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:3.0:1773941549.821427:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e29bb lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:2.0:1773941549.821429:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291487104, portal 25 00000100:00000001:2.0:1773941549.821435:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00000001:3.0:1773941549.821437:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000100:00000001:2.0:1773941549.821437:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330377898624 : -113743331652992 : ffff988d111f0a80) 00010000:00000001:3.0:1773941549.821440:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1773941549.821444:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000040:2.0:1773941549.821445:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d111f0a80 x1860112291487104/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941559 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000040:3.0:1773941549.821446:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030c00 refcount=5 00000020:00000001:3.0:1773941549.821451:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090851328 : -113743618700288 : ffff988d00030c00) 00010000:00000001:3.0:1773941549.821456:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090851328 : -113743618700288 : ffff988d00030c00) 00010000:00000001:3.0:1773941549.821462:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000100:00000001:2.0:1773941549.821462:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941549.821465:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:2.0:1773941549.821468:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516a18. 00000100:00000200:2.0:1773941549.821472:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291487104, offset 0 00010000:00010000:3.0:1773941549.821473:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e29bb lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941549.821480:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 10000000:00000001:3.0:1773941549.821482:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:3.0:1773941549.821485:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e29bb lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941549.821488:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 10000000:01000000:3.0:1773941549.821495:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:2.0:1773941549.821498:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 10000000:00000040:3.0:1773941549.821499:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:3.0:1773941549.821502:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941549.821505:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000400:00000200:2.0:1773941549.821505:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:3.0:1773941549.821508:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:2.0:1773941549.821510:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00000001:3.0:1773941549.821513:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:3.0:1773941549.821515:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:3.0:1773941549.821516:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941549.821519:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941549.821522:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000020:00000040:3.0:1773941549.821523:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d00030c00 with handle 0x8e86767622e29bb from hash 00010000:00000001:3.0:1773941549.821527:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000400:00000200:2.0:1773941549.821528:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:3.0:1773941549.821530:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:2.0:1773941549.821533:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[82, 83] 00010000:00000001:3.0:1773941549.821534:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941549.821537:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:3.0:1773941549.821539:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000400:00000200:2.0:1773941549.821539:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00010000:00000001:3.0:1773941549.821542:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000400:00000200:2.0:1773941549.821546:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [194:194:256:4294967295] 192.168.204.114@tcp LPNI seq info [84:84:8:4294967295] 00010000:00000001:3.0:1773941549.821547:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941549.821550:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941549.821552:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941549.821555:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 10000000:00000001:3.0:1773941549.821556:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941549.821560:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e29bb lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941549.821569:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:2.0:1773941549.821570:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00010000:00000001:3.0:1773941549.821571:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941549.821575:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:3.0:1773941549.821576:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000800:00000200:2.0:1773941549.821577:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00010000:00000001:3.0:1773941549.821579:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941549.821581:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000010:2.0:1773941549.821583:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0db900. 00010000:00000001:3.0:1773941549.821588:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000200:2.0:1773941549.821592:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00010000:00010000:3.0:1773941549.821593:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e29bb lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941549.821601:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00010000:00000040:3.0:1773941549.821602:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d21ec1400 count: 0 00000800:00000200:2.0:1773941549.821606:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0db900 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:3.0:1773941549.821617:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941549.821620:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941549.821621:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941549.821623:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000400:00000200:2.0:1773941549.821626:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000040:3.0:1773941549.821631:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10835180 x1860112291486976/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941565 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941549.821631:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294941417 00000100:00000001:2.0:1773941549.821640:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294941415 00000100:00000001:3.0:1773941549.821641:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941549.821645:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1773941549.821647:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294941417 02000000:00000010:3.0:1773941549.821648:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d02ce6000. 02000000:00000001:3.0:1773941549.821653:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1773941549.821653:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294941415 00000020:00000001:3.0:1773941549.821657:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:2.0:1773941549.821658:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000040:3.0:1773941549.821659:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941549.821663:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941549.821664:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941549.821668:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065c5a00. 00000100:00000001:2.0:1773941549.821669:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941549.821674:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1773941549.821677:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941549.821680:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941549.821683:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10835180. 00000100:00000001:2.0:1773941549.821683:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941549.821687:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1773941549.821689:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941549.821690:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941549.821692:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941549.821693:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 10000000:00000001:3.0:1773941549.821697:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941549.821698:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 10000000:01000000:3.0:1773941549.821702:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000100:00000001:2.0:1773941549.821703:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941549.821707:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:3.0:1773941549.821711:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d03ec7540. 00000020:00000010:3.0:1773941549.821715:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d1a1c8c00. 00000020:00000010:3.0:1773941549.821720:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d1a1c8800. 00000040:00000010:3.0:1773941549.821724:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d1a1c9800. 00000020:00000010:3.0:1773941549.821730:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d08995000. 00000800:00000200:1.0:1773941549.821730:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941549.821744:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0db900. 00000020:00000010:3.0:1773941549.821745:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d1a1c8200. 80000000:00000010:3.0:1773941549.821761:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d111f7c00. 00000400:00000200:1.0:1773941549.821762:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00040000:00000010:3.0:1773941549.821777:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d1a1c8600. 00000400:00000200:1.0:1773941549.821783:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 20000000:00000010:3.0:1773941549.821787:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d1a1c8e00. 00000400:00000200:1.0:1773941549.821788:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516a18 00000400:00000010:1.0:1773941549.821790:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516a18. 00000100:00000001:1.0:1773941549.821799:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941549.821804:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d111f0a80 x1860112291487104/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941559 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941549.821824:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941549.821827:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d111f0a80 x1860112291487104/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941559 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941549.821833:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941549.821837:0:2393:0:(events.c:68:request_out_callback()) Process leaving 10000000:00000040:3.0:1773941549.821839:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:3.0:1773941549.821843:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:3.0:1773941549.821848:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d1a1c8800. 00000040:00000010:3.0:1773941549.821852:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d1a1c9800. 00000020:00000010:3.0:1773941549.821857:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d08995000. 00000020:00000010:3.0:1773941549.821861:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d1a1c8200. 80000000:00000010:3.0:1773941549.821863:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d111f7c00. 00040000:00000010:3.0:1773941549.821866:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d1a1c8600. 20000000:00000010:3.0:1773941549.821869:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d1a1c8e00. 00000020:00000010:3.0:1773941549.821871:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d1a1c8c00. 10000000:00000010:3.0:1773941549.821873:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d03ec7540. 10000000:01000000:3.0:1773941549.821876:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:3.0:1773941549.821879:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:3.0:1773941549.821881:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:3.0:1773941549.821883:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:3.0:1773941549.821884:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:3.0:1773941549.821886:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:3.0:1773941549.821887:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941549.823091:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941549.823100:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941549.823108:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941549.823110:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941549.823114:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941549.823117:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941549.823121:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941549.823124:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941549.823126:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941549.823129:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324605d80 00000400:00000200:0.0:1773941549.823132:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941549.823135:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xe09 [1] + 0 00000800:00000001:0.0:1773941549.823139:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941549.823145:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941549.823146:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941549.823149:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941549.823152:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941549.823156:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d111f0a80 x1860112291487104/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941559 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941549.823164:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d111f0a80 x1860112291487104/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941559 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941549.823180:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941549.823185:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941549.823187:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941549.823240:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941549.823243:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941549.823258:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941549.823259:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941549.823269:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941549.823273:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941549.823274:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941549.823277:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941549.823282:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d111f0a80 x1860112291487104/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941559 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941549.823291:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d111f0a80 x1860112291487104/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941559 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941549.823297:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941549.823298:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941549.823299:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941549.823301:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941549.823302:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941549.823303:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941549.823305:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941549.823307:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=50 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941549.823310:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941549.823312:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941549.823313:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941549.823315:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941549.823317:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941549.823319:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941549.823322:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d111f0a80 x1860112291487104/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941559 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941549.823326:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941549.823328:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941549.823329:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941549.823331:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941549.823336:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941549.823338:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941549.823339:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941549.823340:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941549.823342:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941549.823344:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941549.823348:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:2.0:1773941549.823352:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941549 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941549.823354:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941549.823360:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:2.0:1773941549.823362:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941549.823363:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941549.823365:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941549.823366:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941549.823369:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:2.0:1773941549.823371:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941549.823373:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941549.823376:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941549.823379:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:2.0:1773941549.823381:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941549.823579:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941549.823582:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941549.823589:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d111f0a80 x1860112291487104/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941559 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941549.823597:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d111f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291487104:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941549.823597:0:14759:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:3.0:1773941549.823603:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941549.823606:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1773941549.823608:0:14759:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00000040:3.0:1773941549.823610:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d111f0a80 x1860112291487104/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941559 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941549.823612:0:14759:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941549.823616:0:14759:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941549.823618:0:14759:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:2.0:1773941549.823623:0:14759:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:3.0:1773941549.823624:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941549.823627:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:2.0:1773941549.823627:0:14759:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000010:3.0:1773941549.823631:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a7c00. 00000001:00000001:2.0:1773941549.823632:0:14759:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941549.823635:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:2.0:1773941549.823635:0:14759:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941549.823637:0:14759:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941549.823638:0:14759:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941549.823639:0:14759:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941549.823641:0:14759:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000001:3.0:1773941549.823644:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000100:00080000:2.0:1773941549.823645:0:14759:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000020:00000040:3.0:1773941549.823646:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941549.823650:0:14759:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000020:00000001:3.0:1773941549.823652:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941549.823653:0:14759:0:(recover.c:147:ptlrpc_resend()) Process entered 02000000:00000010:3.0:1773941549.823655:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d03523000. 00000100:00000001:2.0:1773941549.823656:0:14759:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941549.823661:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00080000:2.0:1773941549.823661:0:14759:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 02000000:00000001:3.0:1773941549.823664:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1773941549.823666:0:14759:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000010:3.0:1773941549.823667:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d111f0a80. 10000000:01000000:2.0:1773941549.823668:0:14759:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000100:00000001:3.0:1773941549.823672:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941549.823674:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941549.823678:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:00000040:2.0:1773941549.823678:0:14759:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00000001:3.0:1773941549.823682:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941549.823684:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941549.823686:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941549.823688:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1773941549.823695:0:14759:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:2.0:1773941549.823697:0:14759:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941549.823698:0:14759:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:2.0:1773941549.823703:0:14759:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:2.0:1773941549.823708:0:14759:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941549.823710:0:14759:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941549.823712:0:14759:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941549.823714:0:14759:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941549.823717:0:14759:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:3.1:1773941549.826999:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d21ec1400. 00010000:00000010:3.1:1773941549.827006:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d00030c00. 00000020:00000001:0.0:1773941550.075101:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941550.075106:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941550.075111:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:0.0:1773941550.971092:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1773941550.971098:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941550.971102:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941550.971104:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941550.971106:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941550.971107:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1773941550.971109:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:0.0:1773941550.971111:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:0.0:1773941550.971113:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:0.0:1773941550.971129:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1773941550.971131:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:0.0:1773941550.971136:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10837b80. 00000020:00000040:0.0:1773941550.971140:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941550.971145:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941550.971147:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941550.971152:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941550.971158:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065c5c00. 00000100:00000001:0.0:1773941550.971164:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941550.971168:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1773941550.971171:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1773941550.971178:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d21ec1400. 00010000:00000001:0.0:1773941550.971182:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1773941550.971185:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d00030400. 00000020:00000001:0.0:1773941550.971188:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:0.0:1773941550.971190:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d00030400 with handle 0x8e86767622e29c2 to hash 00000020:00000001:0.0:1773941550.971192:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:0.0:1773941550.971193:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00010000:00000001:0.0:1773941550.971195:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00010000:00000001:0.0:1773941550.971198:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941550.971201:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941550.971204:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e29c2 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941550.971213:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e29c2 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941550.971221:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e29c2 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941550.971239:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1773941550.971241:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941550.971246:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb6600. 00000100:00000001:0.0:1773941550.971248:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330139633152 : -113743569918464 : ffff988d02eb6600) 00000100:00000001:0.0:1773941550.971257:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941550.971258:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367703936 : -113743341847680 : ffff988d10837b80) 00000100:00000001:0.0:1773941550.971262:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941550.971265:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941550.971267:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941550.971268:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941550.971277:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10837b80 x1860112291487232/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941550.971284:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941550.971285:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941550.971288:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941550.971291:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941550.971292:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941550.971295:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10837b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291487232:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941550.971298:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941550.971302:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941550.971304:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941550.971306:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941550.971309:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d02ce7000. 02000000:00000001:0.0:1773941550.971311:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941550.971317:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941550.971319:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0014b370. 00000400:00000010:0.0:1773941550.971325:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833088. 00000100:00000200:0.0:1773941550.971331:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291487232, portal 25 00000100:00000001:0.0:1773941550.971332:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941550.971333:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367703936 : -113743341847680 : ffff988d10837b80) 00000100:00000040:0.0:1773941550.971337:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10837b80 x1860112291487232/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941566 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941550.971342:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941550.971344:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941550.971346:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833bb0. 00000100:00000200:0.0:1773941550.971348:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291487232, offset 0 00000400:00000200:0.0:1773941550.971353:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941550.971364:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941550.971370:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941550.971374:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941550.971378:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941550.971381:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941550.971383:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941550.971385:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[84, 83] 00000400:00000200:0.0:1773941550.971388:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:0.0:1773941550.971391:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [195:195:256:4294967295] 192.168.204.124@tcp LPNI seq info [85:85:8:4294967295] 00000400:00000200:0.0:1773941550.971394:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941550.971402:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:0.0:1773941550.971406:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:0.0:1773941550.971410:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a20bd00. 00000800:00000200:0.0:1773941550.971415:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:0.0:1773941550.971421:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:0.0:1773941550.971424:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a20bd00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1773941550.971438:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941550.971440:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941550.971443:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294942567 00000100:00000001:0.0:1773941550.971448:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294941417 00000100:00000001:0.0:1773941550.971451:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294942567 00000100:00000001:0.0:1773941550.971455:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294941417 00000100:00000001:0.0:1773941550.971457:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941550.971459:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941550.971461:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941550.971463:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1773941550.971465:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6600 going to sleep for 11 seconds 00000100:00000001:0.0:1773941550.971466:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941550.971469:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941550.971542:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941550.971548:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a20bd00. 00000400:00000200:1.0:1773941550.971553:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941550.971562:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941550.971567:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833bb0 00000400:00000010:1.0:1773941550.971571:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833bb0. 00000100:00000001:1.0:1773941550.971575:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941550.971580:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10837b80 x1860112291487232/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941566 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941550.971592:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941550.971596:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d10837b80 x1860112291487232/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941566 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941550.971603:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941550.971605:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941550.972806:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941550.972820:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941550.972832:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941550.972835:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941550.972841:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941550.972849:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941550.972857:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941550.972862:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941550.972867:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941550.972872:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324605e00 00000400:00000200:0.0:1773941550.972876:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941550.972882:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xe11 [1] + 224 00000800:00000001:0.0:1773941550.972890:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941550.972899:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941550.972902:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941550.972906:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941550.972911:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941550.972916:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10837b80 x1860112291487232/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941566 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941550.972926:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d10837b80 x1860112291487232/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941566 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941550.972942:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941550.972949:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941550.972952:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941550.972994:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941550.973005:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941550.973007:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941550.973010:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0014b370. 00000400:00000200:3.0:1773941550.973015:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833088 00000400:00000010:3.0:1773941550.973017:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833088. 00000100:00000001:3.0:1773941550.973020:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941550.973029:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10837b80 x1860112291487232/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941566 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941550.973049:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10837b80 x1860112291487232/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941566 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941550.973055:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941550.973057:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941550.973059:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941550.973062:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941550.973065:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941550.973067:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941550.973069:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941550.973076:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=51 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941550.973079:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941550.973081:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:3.0:1773941550.973084:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:3.0:1773941550.973087:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941550.973103:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:3.0:1773941550.973105:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:3.0:1773941550.973110:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1773941550.973114:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1773941550.973117:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:3.0:1773941550.973119:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941550.973120:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:3.0:1773941550.973122:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:3.0:1773941550.973125:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d10837b80 x1860112291487232/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941566 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941550.973130:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941550.973131:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941550.973133:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941550.973136:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1773941550.973138:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:3.0:1773941550.973139:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941550.973141:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941550.973142:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941550.973142:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:3.0:1773941550.973143:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941550.973144:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941550.973146:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1773941550.973148:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1773941550.973150:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1773941550.973153:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 272 00000400:00000200:3.0:1773941550.973159:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941550.973163:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:3.0:1773941550.973167:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:3.0:1773941550.973170:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941550.973173:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:3.0:1773941550.973174:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:3.0:1773941550.973177:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941550.973182:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941550.973185:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941550.973187:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941550.973189:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941550.973191:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:3.0:1773941550.973194:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:3.0:1773941550.973195:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941550.973196:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:3.0:1773941550.973197:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:3.0:1773941550.973200:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:3.0:1773941550.973203:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:3.0:1773941550.973205:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941550.973208:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1773941550.973210:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941550.973212:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941550.973214:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:3.0:1773941550.973215:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941550.973220:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d111f0a80. 00000020:00000040:3.0:1773941550.973222:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:3.0:1773941550.973239:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1773941550.973241:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941550.973244:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941550.973246:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941550.973261:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941550.973266:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d03523000. 00000100:00000001:3.0:1773941550.973271:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941550.973276:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d111f0a80 x1860112291487360/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1773941550.973283:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941550.973286:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941550.973290:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b933c3a] to pc [ptlrpcd_rcv+-1] req@ffff988d111f0a80 x1860112291487360/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941550.973315:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941550.973317:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:3.0:1773941550.973319:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941550.973321:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941550.973323:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941550.973325:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1773941550.973327:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6600 going to sleep for 1 seconds 00000100:00000001:3.0:1773941550.973329:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941550.973331:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941550.973336:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10837b80 x1860112291487232/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941566 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941550.973341:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941550.973344:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941550.973346:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941550.973346:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941550.973351:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941550.973352:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941550.973355:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941550.973356:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10837b80 x1860112291487232/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941566 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941550.973363:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d111f0a80 x1860112291487360/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941550.973367:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10837b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291487232:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941550.973377:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941550.973382:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941550.973382:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941550.973385:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941550.973386:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:3.0:1773941550.973390:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1773941550.973392:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000040:3.0:1773941550.973396:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10837b80 x1860112291487232/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941566 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1773941550.973397:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941550.973402:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941550.973409:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d111f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291487360:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941550.973411:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941550.973416:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb6600. 00000100:00000001:2.0:1773941550.973417:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1773941550.973420:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 02000000:00000001:2.0:1773941550.973421:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1773941550.973423:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:2.0:1773941550.973424:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941550.973428:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 02000000:00000001:2.0:1773941550.973428:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941550.973434:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a5000. 02000000:00000001:2.0:1773941550.973437:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941550.973439:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1773941550.973441:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:2.0:1773941550.973443:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000020:00000040:3.0:1773941550.973444:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030400 refcount=4 00000020:00000001:3.0:1773941550.973448:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00000400:00000010:2.0:1773941550.973450:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00010000:00000001:3.0:1773941550.973451:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00000400:00000010:2.0:1773941550.973458:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00010000:00010000:3.0:1773941550.973459:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e29c2 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:2.0:1773941550.973466:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291487360, portal 25 00010000:00000001:3.0:1773941550.973468:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941550.973469:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941550.973472:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330377898624 : -113743331652992 : ffff988d111f0a80) 00010000:00010000:3.0:1773941550.973475:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e29c2 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941550.973479:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d111f0a80 x1860112291487360/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941560 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:3.0:1773941550.973483:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:3.0:1773941550.973485:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e29c2 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941550.973502:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941550.973505:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941550.973507:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00010000:3.0:1773941550.973509:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e29c2 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941550.973514:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00000001:3.0:1773941550.973520:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941550.973522:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941550.973524:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000010:2.0:1773941550.973526:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516cc0. 00010000:00010000:3.0:1773941550.973527:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e29c2 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:2.0:1773941550.973533:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291487360, offset 0 00010000:00010000:3.0:1773941550.973535:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d00030400 already unused, calling callback (ffffffffc12b4c20) 10000000:00000001:3.0:1773941550.973538:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:3.0:1773941550.973542:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e29c2 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941550.973542:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00000001:3.0:1773941550.973549:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000400:00000200:2.0:1773941550.973549:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941550.973555:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:3.0:1773941550.973558:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1773941550.973560:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:2.0:1773941550.973561:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000040:3.0:1773941550.973562:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030400 refcount=5 00000020:00000001:3.0:1773941550.973566:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00000400:00000200:2.0:1773941550.973568:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00000001:3.0:1773941550.973570:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00000400:00000200:2.0:1773941550.973573:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:3.0:1773941550.973574:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:3.0:1773941550.973579:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e29c2 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941550.973579:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941550.973587:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[84, 85] 10000000:00000001:3.0:1773941550.973592:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:2.0:1773941550.973594:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 10000000:00010000:3.0:1773941550.973596:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e29c2 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941550.973601:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [196:196:256:4294967295] 192.168.204.114@tcp LPNI seq info [86:86:8:4294967295] 10000000:01000000:3.0:1773941550.973609:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:2.0:1773941550.973610:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 10000000:00000040:3.0:1773941550.973615:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:3.0:1773941550.973617:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941550.973620:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:3.0:1773941550.973623:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:2.0:1773941550.973626:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00010000:00000001:3.0:1773941550.973628:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:3.0:1773941550.973630:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000800:00000200:2.0:1773941550.973634:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00010000:00000001:3.0:1773941550.973635:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941550.973638:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1773941550.973642:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d00030400 with handle 0x8e86767622e29c2 from hash 00000800:00000010:2.0:1773941550.973642:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0db000. 00010000:00000001:3.0:1773941550.973646:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:3.0:1773941550.973648:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941550.973649:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941550.973650:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:3.0:1773941550.973651:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:3.0:1773941550.973653:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000800:00000200:2.0:1773941550.973654:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00010000:00000001:3.0:1773941550.973657:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941550.973659:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941550.973661:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941550.973663:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941550.973667:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00010000:00010000:3.0:1773941550.973668:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e29c2 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941550.973673:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0db000 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:3.0:1773941550.973682:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941550.973683:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941550.973684:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:3.0:1773941550.973686:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000100:00000001:2.0:1773941550.973686:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941550.973687:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941550.973690:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941550.973692:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:2.0:1773941550.973694:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:3.0:1773941550.973695:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e29c2 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941550.973698:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294942569 00010000:00000040:3.0:1773941550.973707:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d21ec1400 count: 0 00000100:00000001:2.0:1773941550.973708:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294942567 00000100:00000001:2.0:1773941550.973713:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294942569 00000100:00000001:2.0:1773941550.973719:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294942567 00000100:00000001:2.0:1773941550.973723:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:3.0:1773941550.973724:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941550.973726:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941550.973727:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941550.973730:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941550.973735:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10837b80 x1860112291487232/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941566 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941550.973737:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941550.973743:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941550.973745:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941550.973748:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941550.973751:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d02ce7000. 00000100:00000001:2.0:1773941550.973752:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:3.0:1773941550.973754:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1773941550.973755:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000020:00000001:3.0:1773941550.973758:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:2.0:1773941550.973759:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:3.0:1773941550.973762:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941550.973762:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941550.973766:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941550.973767:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941550.973769:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941550.973771:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065c5c00. 02000000:00000001:3.0:1773941550.973779:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000800:00000200:1.0:1773941550.973780:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 02000000:00000001:3.0:1773941550.973785:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941550.973792:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10837b80. 00000800:00000010:1.0:1773941550.973793:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0db000. 00000100:00000001:3.0:1773941550.973795:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941550.973799:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:3.0:1773941550.973802:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:3.0:1773941550.973804:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000400:00000200:1.0:1773941550.973805:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 10000000:00000001:3.0:1773941550.973808:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:3.0:1773941550.973811:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d03ec7540. 00000400:00000200:1.0:1773941550.973814:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000020:00000010:3.0:1773941550.973818:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d1a1c8c00. 00000400:00000200:1.0:1773941550.973819:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516cc0 00000400:00000010:1.0:1773941550.973821:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516cc0. 00000100:00000001:1.0:1773941550.973824:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941550.973830:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d111f0a80 x1860112291487360/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941560 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941550.973837:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941550.973839:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d111f0a80 x1860112291487360/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941560 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941550.973854:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941550.973856:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:3.0:1773941550.973882:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d1a1c8e00. 00000040:00000010:3.0:1773941550.973885:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d1a1c8600. 00000020:00000010:3.0:1773941550.973890:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d08995000. 00000020:00000010:3.0:1773941550.973894:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d1a1c8200. 80000000:00000010:3.0:1773941550.973897:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d111f7c00. 00040000:00000010:3.0:1773941550.973901:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d1a1c9800. 20000000:00000010:3.0:1773941550.973905:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d1a1c8800. 10000000:00000040:3.0:1773941550.973909:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:3.0:1773941550.973911:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:3.0:1773941550.973915:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d1a1c8e00. 00000040:00000010:3.0:1773941550.973918:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d1a1c8600. 00000020:00000010:3.0:1773941550.973921:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d08995000. 00000020:00000010:3.0:1773941550.973924:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d1a1c8200. 80000000:00000010:3.0:1773941550.973927:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d111f7c00. 00040000:00000010:3.0:1773941550.973929:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d1a1c9800. 20000000:00000010:3.0:1773941550.973932:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d1a1c8800. 00000020:00000010:3.0:1773941550.973934:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d1a1c8c00. 10000000:00000010:3.0:1773941550.973936:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d03ec7540. 10000000:01000000:3.0:1773941550.973939:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:3.0:1773941550.973942:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:3.0:1773941550.973943:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:3.0:1773941550.973945:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:3.0:1773941550.973947:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:3.0:1773941550.973949:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:3.0:1773941550.973950:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941550.975217:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941550.975225:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941550.975237:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941550.975241:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941550.975246:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941550.975251:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941550.975256:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941550.975261:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941550.975265:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941550.975268:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324605e80 00000400:00000200:0.0:1773941550.975272:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941550.975285:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xe21 [1] + 0 00000800:00000001:0.0:1773941550.975291:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941550.975299:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941550.975302:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941550.975306:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941550.975311:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941550.975316:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d111f0a80 x1860112291487360/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941560 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941550.975327:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d111f0a80 x1860112291487360/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941560 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941550.975346:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941550.975351:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941550.975355:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941550.975376:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941550.975379:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941550.975382:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941550.975383:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941550.975385:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941550.975389:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941550.975390:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941550.975392:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941550.975397:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d111f0a80 x1860112291487360/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941560 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941550.975405:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d111f0a80 x1860112291487360/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941560 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941550.975410:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941550.975411:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941550.975412:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941550.975414:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941550.975415:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941550.975416:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941550.975417:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941550.975420:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=51 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941550.975422:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941550.975424:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941550.975425:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941550.975427:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941550.975428:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941550.975431:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941550.975434:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d111f0a80 x1860112291487360/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941560 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941550.975448:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941550.975450:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941550.975451:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941550.975453:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941550.975458:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941550.975459:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941550.975460:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941550.975461:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941550.975462:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941550.975464:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941550.975466:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:2.0:1773941550.975468:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941550 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941550.975471:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941550.975477:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:2.0:1773941550.975478:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941550.975479:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941550.975481:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941550.975482:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941550.975485:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:2.0:1773941550.975487:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941550.975488:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941550.975491:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941550.975494:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:2.0:1773941550.975495:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941550.975718:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941550.975724:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941550.975732:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d111f0a80 x1860112291487360/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941560 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941550.975735:0:14760:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:3.0:1773941550.975746:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d111f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291487360:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00080000:2.0:1773941550.975746:0:14760:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:2.0:1773941550.975752:0:14760:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941550.975755:0:14760:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:3.0:1773941550.975756:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 10000000:01000000:2.0:1773941550.975757:0:14760:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:2.0:1773941550.975760:0:14760:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941550.975762:0:14760:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:3.0:1773941550.975768:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000001:00000001:2.0:1773941550.975773:0:14760:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:2.0:1773941550.975777:0:14760:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000040:3.0:1773941550.975778:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d111f0a80 x1860112291487360/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941560 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000001:00000001:2.0:1773941550.975778:0:14760:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941550.975781:0:14760:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941550.975786:0:14760:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941550.975790:0:14760:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941550.975795:0:14760:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:3.0:1773941550.975799:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1773941550.975800:0:14760:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941550.975803:0:14760:0:(recover.c:147:ptlrpc_resend()) Process entered 02000000:00000001:3.0:1773941550.975805:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1773941550.975806:0:14760:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941550.975809:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a5000. 00000100:00080000:2.0:1773941550.975810:0:14760:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941550.975814:0:14760:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:3.0:1773941550.975815:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 10000000:01000000:2.0:1773941550.975816:0:14760:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000020:00000001:3.0:1773941550.975818:0:2400:0:(genops.c:1236:class_import_put()) Process entered 10000000:00000040:2.0:1773941550.975819:0:14760:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000020:00000040:3.0:1773941550.975821:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941550.975824:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941550.975826:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d03523000. 02000000:00000001:3.0:1773941550.975828:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941550.975829:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941550.975830:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d111f0a80. 00000100:00000001:3.0:1773941550.975833:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941550.975835:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1773941550.975835:0:14760:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:00000001:3.0:1773941550.975838:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:2.0:1773941550.975839:0:14760:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941550.975843:0:14760:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941550.975844:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941550.975848:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:02000000:2.0:1773941550.975848:0:14760:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:3.0:1773941550.975854:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941550.975856:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941550.975858:0:14760:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941550.975861:0:14760:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941550.975863:0:14760:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941550.975865:0:14760:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941550.975867:0:14760:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:3.1:1773941550.981018:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d21ec1400. 00010000:00000010:3.1:1773941550.981029:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d00030400. 00000020:00000001:2.0:1773941551.099202:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941551.099209:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941551.099215:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941551.355049:0:461:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (279) 00000020:00000001:0.0:1773941552.123128:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941552.123133:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941552.123138:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1773941552.251067:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294942569, at 4294942592 00000100:00000010:0.0:1773941552.251081:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580b40. 00000100:00000040:1.0:1773941552.251094:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294942569, at 4294942592 00000100:00000010:1.0:1773941552.251106:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580300. 10000000:00000001:0.0:1773941552.443058:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1773941552.443065:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941552.443069:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941552.443070:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941552.443072:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941552.443073:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1773941552.443075:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:0.0:1773941552.443077:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:0.0:1773941552.443079:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:0.0:1773941552.443081:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1773941552.443082:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:0.0:1773941552.443093:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10835880. 00000020:00000040:0.0:1773941552.443097:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941552.443101:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941552.443104:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941552.443109:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941552.443128:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065c4a00. 00000100:00000001:0.0:1773941552.443133:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941552.443137:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1773941552.443140:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1773941552.443146:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d21ec1800. 00010000:00000001:0.0:1773941552.443151:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1773941552.443155:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d00030400. 00000020:00000001:0.0:1773941552.443158:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:0.0:1773941552.443160:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d00030400 with handle 0x8e86767622e29c9 to hash 00000020:00000001:0.0:1773941552.443161:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:0.0:1773941552.443163:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00010000:00000001:0.0:1773941552.443165:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00010000:00000001:0.0:1773941552.443168:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941552.443171:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941552.443174:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e29c9 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941552.443183:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e29c9 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941552.443192:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e29c9 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941552.443197:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1773941552.443214:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941552.443219:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb6840. 00000100:00000001:0.0:1773941552.443221:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330139633728 : -113743569917888 : ffff988d02eb6840) 00000100:00000001:0.0:1773941552.443223:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941552.443224:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367694976 : -113743341856640 : ffff988d10835880) 00000100:00000001:0.0:1773941552.443229:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941552.443232:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941552.443250:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941552.443252:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941552.443262:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10835880 x1860112291487488/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941552.443278:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941552.443280:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941552.443284:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941552.443286:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941552.443288:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941552.443292:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10835880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291487488:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941552.443296:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941552.443301:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941552.443303:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941552.443305:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941552.443310:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d02ce7400. 02000000:00000001:0.0:1773941552.443311:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941552.443320:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941552.443323:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0014b840. 00000400:00000010:0.0:1773941552.443353:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833e58. 00000100:00000200:0.0:1773941552.443358:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291487488, portal 25 00000100:00000001:0.0:1773941552.443360:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941552.443361:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367694976 : -113743341856640 : ffff988d10835880) 00000100:00000040:0.0:1773941552.443365:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10835880 x1860112291487488/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941568 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941552.443371:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941552.443373:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941552.443374:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833f68. 00000100:00000200:0.0:1773941552.443376:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291487488, offset 0 00000400:00000200:0.0:1773941552.443381:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941552.443389:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941552.443395:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941552.443399:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941552.443402:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941552.443406:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941552.443408:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941552.443414:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[86, 85] 00000400:00000200:0.0:1773941552.443416:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:0.0:1773941552.443420:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [197:197:256:4294967295] 192.168.204.124@tcp LPNI seq info [87:87:8:4294967295] 00000400:00000200:0.0:1773941552.443423:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941552.443431:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:0.0:1773941552.443436:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:0.0:1773941552.443440:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a20bd00. 00000800:00000200:0.0:1773941552.443445:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:0.0:1773941552.443451:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:0.0:1773941552.443454:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a20bd00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1773941552.443468:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941552.443472:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941552.443474:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294944039 00000100:00000010:0.0:1773941552.443477:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580b40. 00000100:00000001:0.0:1773941552.443482:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1773941552.443490:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294944039 00000100:00000010:0.0:1773941552.443493:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580000. 00000100:00000001:0.0:1773941552.443496:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1773941552.443499:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941552.443501:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941552.443503:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941552.443506:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1773941552.443508:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6840 going to sleep for 11 seconds 00000100:00000001:0.0:1773941552.443510:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941552.443513:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941552.443604:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941552.443611:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a20bd00. 00000400:00000200:1.0:1773941552.443618:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941552.443625:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941552.443631:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833f68 00000400:00000010:1.0:1773941552.443634:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833f68. 00000100:00000001:1.0:1773941552.443639:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941552.443646:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10835880 x1860112291487488/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941568 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941552.443657:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941552.443660:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d10835880 x1860112291487488/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941568 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941552.443666:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941552.443667:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941552.444954:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941552.444982:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941552.444984:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941552.444986:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941552.444990:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941552.444995:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941552.444999:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941552.445008:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941552.445012:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941552.445015:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324605f00 00000400:00000200:0.0:1773941552.445018:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941552.445022:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xe29 [1] + 224 00000800:00000001:0.0:1773941552.445027:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941552.445034:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941552.445037:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941552.445040:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941552.445045:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941552.445050:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10835880 x1860112291487488/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941568 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941552.445059:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d10835880 x1860112291487488/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941568 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941552.445073:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941552.445079:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941552.445082:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941552.445111:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941552.445118:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941552.445119:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941552.445121:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0014b840. 00000400:00000200:3.0:1773941552.445125:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833e58 00000400:00000010:3.0:1773941552.445127:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833e58. 00000100:00000001:3.0:1773941552.445129:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941552.445138:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10835880 x1860112291487488/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941568 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941552.445147:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10835880 x1860112291487488/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941568 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941552.445152:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941552.445164:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941552.445166:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941552.445168:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941552.445176:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941552.445178:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941552.445180:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941552.445185:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=53 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941552.445188:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941552.445190:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:3.0:1773941552.445194:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:3.0:1773941552.445196:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941552.445211:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:3.0:1773941552.445214:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:3.0:1773941552.445218:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1773941552.445222:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1773941552.445225:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:3.0:1773941552.445227:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941552.445228:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:3.0:1773941552.445230:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:3.0:1773941552.445233:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d10835880 x1860112291487488/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941568 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941552.445238:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941552.445239:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941552.445240:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941552.445242:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1773941552.445244:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:3.0:1773941552.445246:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941552.445247:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941552.445248:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941552.445249:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:3.0:1773941552.445250:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941552.445251:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941552.445265:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1773941552.445266:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1773941552.445268:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1773941552.445270:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 274 00000400:00000200:3.0:1773941552.445278:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941552.445282:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:3.0:1773941552.445286:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:3.0:1773941552.445289:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941552.445291:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:3.0:1773941552.445293:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:3.0:1773941552.445298:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941552.445300:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941552.445303:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941552.445305:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941552.445308:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941552.445311:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:3.0:1773941552.445315:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:3.0:1773941552.445317:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941552.445319:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:3.0:1773941552.445321:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:3.0:1773941552.445324:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:3.0:1773941552.445327:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:3.0:1773941552.445330:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941552.445333:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1773941552.445337:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941552.445340:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941552.445342:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:3.0:1773941552.445344:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941552.445348:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d111f0a80. 00000020:00000040:3.0:1773941552.445351:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:3.0:1773941552.445355:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1773941552.445356:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941552.445359:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941552.445361:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941552.445363:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941552.445367:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d03523000. 00000100:00000001:3.0:1773941552.445381:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941552.445389:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d111f0a80 x1860112291487616/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1773941552.445399:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941552.445403:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941552.445408:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b933c3a] to pc [ptlrpcd_rcv+-1] req@ffff988d111f0a80 x1860112291487616/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941552.445430:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941552.445434:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:3.0:1773941552.445435:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941552.445437:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941552.445438:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941552.445439:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1773941552.445441:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6840 going to sleep for 1 seconds 00000100:00000001:3.0:1773941552.445443:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941552.445445:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941552.445449:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10835880 x1860112291487488/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941568 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941552.445455:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941552.445456:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941552.445460:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941552.445461:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941552.445462:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941552.445463:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1773941552.445466:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941552.445471:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d111f0a80 x1860112291487616/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941552.445472:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10835880 x1860112291487488/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941568 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941552.445495:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941552.445498:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941552.445499:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10835880 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291487488:192.168.204.114@tcp:101:ll_cfg_requeue.0 02000000:00000001:2.0:1773941552.445501:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1773941552.445511:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1773941552.445511:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941552.445515:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:2.0:1773941552.445516:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941552.445518:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:3.0:1773941552.445520:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941552.445522:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10835880 x1860112291487488/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941568 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941552.445528:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d111f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291487616:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941552.445535:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941552.445539:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941552.445541:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941552.445544:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941552.445544:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:3.0:1773941552.445548:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb6840. 02000000:00000010:2.0:1773941552.445549:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a4c00. 00000100:00000001:3.0:1773941552.445552:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 02000000:00000001:2.0:1773941552.445552:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941552.445555:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000200:2.0:1773941552.445559:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:3.0:1773941552.445561:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000400:00000010:2.0:1773941552.445563:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00010000:00000001:3.0:1773941552.445576:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000010:2.0:1773941552.445577:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000020:00000001:3.0:1773941552.445580:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000200:2.0:1773941552.445582:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291487616, portal 25 00000020:00000040:3.0:1773941552.445583:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030400 refcount=4 00000100:00000001:2.0:1773941552.445587:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000020:00000001:3.0:1773941552.445588:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00000100:00000001:2.0:1773941552.445589:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330377898624 : -113743331652992 : ffff988d111f0a80) 00010000:00000001:3.0:1773941552.445592:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00000100:00000040:2.0:1773941552.445596:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d111f0a80 x1860112291487616/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941562 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:3.0:1773941552.445598:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e29c9 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941552.445616:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941552.445617:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941552.445621:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00010000:3.0:1773941552.445623:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e29c9 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941552.445624:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516330. 00000100:00000200:2.0:1773941552.445629:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291487616, offset 0 00010000:00000001:3.0:1773941552.445634:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000400:00000200:2.0:1773941552.445637:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00010000:3.0:1773941552.445639:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e29c9 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941552.445644:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:3.0:1773941552.445650:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:2.0:1773941552.445650:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:3.0:1773941552.445654:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:2.0:1773941552.445656:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00010000:3.0:1773941552.445658:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e29c9 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941552.445660:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941552.445666:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:3.0:1773941552.445670:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:2.0:1773941552.445671:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:3.0:1773941552.445674:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941552.445676:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[86, 87] 00010000:00000001:3.0:1773941552.445679:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:2.0:1773941552.445681:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00010000:00010000:3.0:1773941552.445683:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e29c9 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941552.445687:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [198:198:256:4294967295] 192.168.204.114@tcp LPNI seq info [88:88:8:4294967295] 00000400:00000200:2.0:1773941552.445694:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941552.445705:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00010000:00010000:3.0:1773941552.445706:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d00030400 already unused, calling callback (ffffffffc12b4c20) 10000000:00000001:3.0:1773941552.445709:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:3.0:1773941552.445711:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e29c9 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941552.445723:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000800:00000200:2.0:1773941552.445723:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00010000:00000001:3.0:1773941552.445725:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1773941552.445728:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000010:2.0:1773941552.445729:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0dbd00. 00000020:00000040:3.0:1773941552.445732:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030400 refcount=5 00000020:00000001:3.0:1773941552.445734:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00000800:00000200:2.0:1773941552.445738:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00010000:00000001:3.0:1773941552.445739:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00010000:00000001:3.0:1773941552.445744:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:3.0:1773941552.445747:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e29c9 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941552.445748:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:2.0:1773941552.445753:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0dbd00 type 1, nob 616 niov 1 nkiov 1 10000000:00000001:3.0:1773941552.445759:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:3.0:1773941552.445762:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e29c9 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:3.0:1773941552.445766:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:3.0:1773941552.445768:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000100:00000001:2.0:1773941552.445768:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941552.445769:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941552.445775:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000400:00000200:2.0:1773941552.445776:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:3.0:1773941552.445777:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000100:00000001:2.0:1773941552.445779:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294944041 00010000:00000001:3.0:1773941552.445781:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:3.0:1773941552.445784:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000100:00000001:2.0:1773941552.445785:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294944039 00010000:00000001:3.0:1773941552.445786:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941552.445789:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941552.445791:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294944041 00000020:00000040:3.0:1773941552.445792:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d00030400 with handle 0x8e86767622e29c9 from hash 00000100:00000001:2.0:1773941552.445797:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294944039 00010000:00000001:3.0:1773941552.445798:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:3.0:1773941552.445811:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941552.445813:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:3.0:1773941552.445814:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941552.445816:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941552.445819:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941552.445822:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000100:00000001:2.0:1773941552.445823:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1773941552.445825:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:3.0:1773941552.445828:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:3.0:1773941552.445831:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941552.445831:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941552.445834:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00010000:00000001:3.0:1773941552.445835:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941552.445839:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:3.0:1773941552.445840:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941552.445841:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941552.445844:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941552.445846:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941552.445847:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941552.445854:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e29c9 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941552.445862:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941552.445865:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941552.445867:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:3.0:1773941552.445869:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:3.0:1773941552.445871:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941552.445872:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941552.445874:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:3.0:1773941552.445877:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e29c9 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:3.0:1773941552.445886:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d21ec1800 count: 0 00000800:00000200:1.0:1773941552.445898:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00000001:3.0:1773941552.445902:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000010:1.0:1773941552.445905:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0dbd00. 00010000:00000001:3.0:1773941552.445907:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941552.445914:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000400:00000200:1.0:1773941552.445915:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000040:3.0:1773941552.445923:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10835880 x1860112291487488/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941568 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000200:1.0:1773941552.445927:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941552.445935:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516330 00000400:00000010:1.0:1773941552.445938:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516330. 00000100:00000001:1.0:1773941552.445942:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941552.445949:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d111f0a80 x1860112291487616/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941562 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941552.445959:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941552.446005:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d111f0a80 x1860112291487616/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941562 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941552.446055:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941552.446059:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941552.446063:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d02ce7400. 00000100:00000001:1.0:1773941552.446063:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941552.446074:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:1.0:1773941552.446074:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:3.0:1773941552.446082:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941552.446084:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941552.446087:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941552.446089:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065c4a00. 02000000:00000001:3.0:1773941552.446094:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941552.446110:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941552.446120:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10835880. 00000100:00000001:3.0:1773941552.446123:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941552.446126:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:3.0:1773941552.446129:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:3.0:1773941552.446131:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:3.0:1773941552.446133:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:3.0:1773941552.446136:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d03ec7540. 00000020:00000010:3.0:1773941552.446140:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d1a1c8c00. 00000020:00000010:3.0:1773941552.446144:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d1a1c8800. 00000040:00000010:3.0:1773941552.446148:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d1a1c9800. 00000020:00000010:3.0:1773941552.446151:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d08995000. 00000020:00000010:3.0:1773941552.446155:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d1a1c8200. 80000000:00000010:3.0:1773941552.446175:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d111f7c00. 00040000:00000010:3.0:1773941552.446216:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d1a1c8600. 20000000:00000010:3.0:1773941552.446219:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d1a1c8e00. 10000000:00000040:3.0:1773941552.446223:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:3.0:1773941552.446224:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:3.0:1773941552.446228:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d1a1c8800. 00000040:00000010:3.0:1773941552.446231:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d1a1c9800. 00000020:00000010:3.0:1773941552.446233:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d08995000. 00000020:00000010:3.0:1773941552.446236:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d1a1c8200. 80000000:00000010:3.0:1773941552.446238:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d111f7c00. 00040000:00000010:3.0:1773941552.446240:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d1a1c8600. 20000000:00000010:3.0:1773941552.446242:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d1a1c8e00. 00000020:00000010:3.0:1773941552.446244:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d1a1c8c00. 10000000:00000010:3.0:1773941552.446246:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d03ec7540. 10000000:01000000:3.0:1773941552.446248:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:3.0:1773941552.446251:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:3.0:1773941552.446252:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:3.0:1773941552.446254:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:3.0:1773941552.446255:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:3.0:1773941552.446256:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:3.0:1773941552.446258:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941552.447686:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941552.447692:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941552.447694:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941552.447696:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941552.447700:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941552.447704:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941552.447710:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941552.447714:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941552.447717:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941552.447721:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324605f80 00000400:00000200:0.0:1773941552.447740:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941552.447745:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xe39 [1] + 0 00000800:00000001:0.0:1773941552.447751:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941552.447757:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941552.447760:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941552.447773:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941552.447779:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941552.447785:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d111f0a80 x1860112291487616/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941562 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941552.447798:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d111f0a80 x1860112291487616/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941562 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941552.447827:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941552.447834:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941552.447838:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941552.447903:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941552.447906:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941552.447911:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941552.447913:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941552.447916:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941552.447922:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941552.447923:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941552.447925:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941552.447930:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d111f0a80 x1860112291487616/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941562 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941552.447945:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d111f0a80 x1860112291487616/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941562 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941552.447994:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941552.447996:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941552.447998:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941552.448000:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941552.448001:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941552.448003:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941552.448005:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941552.448008:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=53 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941552.448010:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941552.448012:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941552.448013:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941552.448015:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941552.448017:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941552.448019:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941552.448023:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d111f0a80 x1860112291487616/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941562 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941552.448028:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941552.448029:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941552.448031:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941552.448033:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941552.448036:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941552.448038:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941552.448038:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941552.448040:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941552.448041:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941552.448043:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941552.448045:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:2.0:1773941552.448047:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941552 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941552.448050:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941552.448054:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:2.0:1773941552.448056:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941552.448057:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941552.448059:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941552.448075:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941552.448078:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:2.0:1773941552.448080:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941552.448082:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941552.448085:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941552.448088:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:2.0:1773941552.448090:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941552.448316:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941552.448332:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941552.448332:0:14761:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00080000:2.0:1773941552.448344:0:14761:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00000040:3.0:1773941552.448351:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d111f0a80 x1860112291487616/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941562 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941552.448351:0:14761:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00100000:3.0:1773941552.448374:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d111f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291487616:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941552.448388:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941552.448393:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941552.448398:0:14761:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000040:3.0:1773941552.448399:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d111f0a80 x1860112291487616/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941562 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:01000000:2.0:1773941552.448401:0:14761:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:2.0:1773941552.448407:0:14761:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941552.448410:0:14761:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:3.0:1773941552.448431:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:2.0:1773941552.448434:0:14761:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941552.448435:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:2.0:1773941552.448438:0:14761:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941552.448440:0:14761:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000010:3.0:1773941552.448441:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a4c00. 00000001:00000001:2.0:1773941552.448443:0:14761:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941552.448447:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 10000000:00000001:2.0:1773941552.448448:0:14761:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941552.448456:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:2.0:1773941552.448457:0:14761:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000040:3.0:1773941552.448468:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00080000:2.0:1773941552.448471:0:14761:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000020:00000001:3.0:1773941552.448473:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941552.448478:0:14761:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 02000000:00000010:3.0:1773941552.448479:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d03523000. 00000100:00000001:2.0:1773941552.448482:0:14761:0:(recover.c:147:ptlrpc_resend()) Process entered 02000000:00000001:3.0:1773941552.448484:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1773941552.448486:0:14761:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941552.448493:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:2.0:1773941552.448494:0:14761:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000010:3.0:1773941552.448497:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d111f0a80. 00000100:00000001:2.0:1773941552.448498:0:14761:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941552.448502:0:14761:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000100:00000001:3.0:1773941552.448504:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 10000000:00000040:2.0:1773941552.448504:0:14761:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00000001:3.0:1773941552.448527:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941552.448531:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1773941552.448536:0:14761:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:00000001:3.0:1773941552.448539:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 10000000:00000001:2.0:1773941552.448539:0:14761:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941552.448543:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941552.448545:0:14761:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941552.448552:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941552.448559:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:02000000:2.0:1773941552.448559:0:14761:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:2.0:1773941552.448573:0:14761:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941552.448576:0:14761:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941552.448585:0:14761:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941552.448587:0:14761:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941552.448589:0:14761:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:3.1:1773941552.451044:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d21ec1800. 00010000:00000010:3.1:1773941552.451053:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d00030400. 00000020:00000001:2.0:1773941553.147161:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941553.147168:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941553.147173:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1773941553.723205:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294944041, at 4294944064 00000100:00000040:1.0:1773941553.723216:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294944041, at 4294944064 00000100:00000010:0.0:1773941553.723236:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580b40. 00000100:00000010:1.0:1773941553.723256:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580000. 10000000:00000001:0.0:1773941553.787052:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1773941553.787061:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941553.787067:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941553.787071:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941553.787073:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941553.787075:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1773941553.787078:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:0.0:1773941553.787081:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:0.0:1773941553.787085:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:0.0:1773941553.787088:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1773941553.787090:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:0.0:1773941553.787104:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10836680. 00000020:00000040:0.0:1773941553.787110:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941553.787116:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941553.787120:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941553.787127:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941553.787136:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065c5e00. 00000100:00000001:0.0:1773941553.787144:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941553.787150:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1773941553.787154:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1773941553.787166:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d21ec1800. 00010000:00000001:0.0:1773941553.787195:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1773941553.787200:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d00030c00. 00000020:00000001:0.0:1773941553.787224:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:0.0:1773941553.787228:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d00030c00 with handle 0x8e86767622e29d0 to hash 00000020:00000001:0.0:1773941553.787231:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:0.0:1773941553.787233:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330090851328 : -113743618700288 : ffff988d00030c00) 00010000:00000001:0.0:1773941553.787238:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330090851328 : -113743618700288 : ffff988d00030c00) 00010000:00000001:0.0:1773941553.787243:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941553.787247:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941553.787254:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e29d0 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941553.787268:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e29d0 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941553.787284:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e29d0 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941553.787291:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1773941553.787294:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941553.787303:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb66c0. 00000100:00000001:0.0:1773941553.787305:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330139633344 : -113743569918272 : ffff988d02eb66c0) 00000100:00000001:0.0:1773941553.787309:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941553.787311:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367698560 : -113743341853056 : ffff988d10836680) 00000100:00000001:0.0:1773941553.787318:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941553.787323:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941553.787326:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941553.787328:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941553.787342:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10836680 x1860112291487744/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941553.787355:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941553.787357:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941553.787362:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941553.787365:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941553.787369:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941553.787380:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10836680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291487744:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941553.787386:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941553.787393:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941553.787396:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941553.787400:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941553.787405:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d02ce7400. 02000000:00000001:0.0:1773941553.787408:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941553.787418:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941553.787423:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0014b7e8. 00000400:00000010:0.0:1773941553.787431:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d108336e8. 00000100:00000200:0.0:1773941553.787440:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291487744, portal 25 00000100:00000001:0.0:1773941553.787443:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941553.787444:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367698560 : -113743341853056 : ffff988d10836680) 00000100:00000040:0.0:1773941553.787451:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10836680 x1860112291487744/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941569 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941553.787462:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941553.787465:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941553.787468:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833d48. 00000100:00000200:0.0:1773941553.787472:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291487744, offset 0 00000400:00000200:0.0:1773941553.787479:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941553.787490:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941553.787499:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941553.787551:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941553.787557:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941553.787562:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941553.787566:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941553.787571:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[88, 87] 00000400:00000200:0.0:1773941553.787576:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:0.0:1773941553.787582:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [199:199:256:4294967295] 192.168.204.124@tcp LPNI seq info [89:89:8:4294967295] 00000400:00000200:0.0:1773941553.787587:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941553.787600:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:0.0:1773941553.787621:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:0.0:1773941553.787629:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a20bd00. 00000800:00000200:0.0:1773941553.787638:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:0.0:1773941553.787649:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:0.0:1773941553.787655:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a20bd00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1773941553.787674:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941553.787680:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941553.787685:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294945383 00000100:00000010:0.0:1773941553.787691:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580b40. 00000100:00000001:0.0:1773941553.787701:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1773941553.787712:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294945383 00000100:00000010:0.0:1773941553.787720:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580c60. 00000100:00000001:0.0:1773941553.787728:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1773941553.787738:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941553.787743:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941553.787749:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941553.787752:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1773941553.787757:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb66c0 going to sleep for 11 seconds 00000100:00000001:0.0:1773941553.787764:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941553.787770:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941553.788000:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941553.788009:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a20bd00. 00000400:00000200:1.0:1773941553.788018:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941553.788026:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941553.788034:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833d48 00000400:00000010:1.0:1773941553.788037:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833d48. 00000100:00000001:1.0:1773941553.788043:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941553.788051:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10836680 x1860112291487744/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941569 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941553.788066:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941553.788071:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d10836680 x1860112291487744/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941569 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941553.788081:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941553.788084:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941553.789822:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941553.789846:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941553.789850:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941553.789854:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941553.789861:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941553.789869:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941553.789879:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941553.789884:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941553.789890:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941553.789895:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324606000 00000400:00000200:0.0:1773941553.789901:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941553.789908:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xe41 [1] + 224 00000800:00000001:0.0:1773941553.789934:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941553.789945:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941553.789948:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941553.789953:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941553.789959:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941553.789997:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10836680 x1860112291487744/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941569 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941553.790012:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d10836680 x1860112291487744/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941569 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941553.790041:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941553.790053:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941553.790058:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941553.790168:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941553.790225:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941553.790229:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941553.790234:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0014b7e8. 00000400:00000200:3.0:1773941553.790244:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d108336e8 00000400:00000010:3.0:1773941553.790248:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d108336e8. 00000100:00000001:3.0:1773941553.790255:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941553.790268:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10836680 x1860112291487744/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941569 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941553.790321:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10836680 x1860112291487744/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941569 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941553.790337:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941553.790341:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941553.790347:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941553.790352:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941553.790356:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941553.790360:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941553.790365:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941553.790376:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=54 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941553.790382:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941553.790388:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:3.0:1773941553.790396:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:3.0:1773941553.790401:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941553.790405:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:3.0:1773941553.790409:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:3.0:1773941553.790415:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:3.0:1773941553.790421:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1773941553.790425:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:3.0:1773941553.790429:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:3.0:1773941553.790431:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:3.0:1773941553.790434:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:3.0:1773941553.790439:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d10836680 x1860112291487744/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941569 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941553.790449:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:3.0:1773941553.790450:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:3.0:1773941553.790453:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941553.790456:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1773941553.790459:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:3.0:1773941553.790462:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941553.790465:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941553.790482:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941553.790485:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:3.0:1773941553.790488:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941553.790491:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941553.790495:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1773941553.790499:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1773941553.790505:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:3.0:1773941553.790512:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 275 00000400:00000200:3.0:1773941553.790523:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941553.790531:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:3.0:1773941553.790539:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:3.0:1773941553.790545:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941553.790551:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:3.0:1773941553.790555:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:3.0:1773941553.790561:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941553.790565:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941553.790569:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941553.790573:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941553.790577:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941553.790582:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:3.0:1773941553.790588:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:3.0:1773941553.790593:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:3.0:1773941553.790597:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:3.0:1773941553.790600:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:3.0:1773941553.790606:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:3.0:1773941553.790613:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:3.0:1773941553.790629:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941553.790634:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1773941553.790639:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941553.790644:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941553.790649:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:3.0:1773941553.790652:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941553.790671:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d111f0a80. 00000020:00000040:3.0:1773941553.790677:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:3.0:1773941553.790683:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1773941553.790686:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941553.790690:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941553.790694:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1773941553.790699:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941553.790706:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d03523000. 00000100:00000001:3.0:1773941553.790714:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941553.790726:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d111f0a80 x1860112291487872/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1773941553.790743:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1773941553.790749:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941553.790758:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b933c3a] to pc [ptlrpcd_rcv+-1] req@ffff988d111f0a80 x1860112291487872/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941553.790797:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941553.790803:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:3.0:1773941553.790807:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941553.790812:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941553.790817:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941553.790820:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1773941553.790824:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb66c0 going to sleep for 1 seconds 00000100:00000001:3.0:1773941553.790828:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941553.790832:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941553.790843:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10836680 x1860112291487744/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941569 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941553.790857:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941553.790863:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941553.790867:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941553.790872:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941553.790883:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10836680 x1860112291487744/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941569 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941553.790899:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941553.790906:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941553.790909:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941553.790920:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d111f0a80 x1860112291487872/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941553.790949:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941553.790954:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941553.790999:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941553.791003:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941553.791007:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941553.791013:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d111f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291487872:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941553.791020:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941553.791023:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941553.791026:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941553.791030:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941553.791039:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a7400. 02000000:00000001:2.0:1773941553.791046:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941553.791050:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10836680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291487744:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000400:00000200:2.0:1773941553.791058:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941553.791067:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000100:00000001:3.0:1773941553.791071:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:2.0:1773941553.791075:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000001:3.0:1773941553.791080:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000200:2.0:1773941553.791084:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291487872, portal 25 00000100:00000001:3.0:1773941553.791090:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:2.0:1773941553.791091:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941553.791099:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330377898624 : -113743331652992 : ffff988d111f0a80) 00000100:00000001:3.0:1773941553.791101:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941553.791110:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d111f0a80 x1860112291487872/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941563 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941553.791118:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10836680 x1860112291487744/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941569 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941553.791137:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941553.791152:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:2.0:1773941553.791158:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516c38. 00000100:00000001:3.0:1773941553.791163:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941553.791169:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb66c0. 00000100:00000001:3.0:1773941553.791208:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000200:2.0:1773941553.791210:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291487872, offset 0 00000100:00000001:3.0:1773941553.791225:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000200:2.0:1773941553.791236:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00000001:3.0:1773941553.791238:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:3.0:1773941553.791248:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:2.0:1773941553.791248:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000020:00000001:3.0:1773941553.791258:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:2.0:1773941553.791264:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000020:00000040:3.0:1773941553.791268:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030c00 refcount=4 00000400:00000200:2.0:1773941553.791272:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:3.0:1773941553.791280:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090851328 : -113743618700288 : ffff988d00030c00) 00000400:00000200:2.0:1773941553.791281:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941553.791291:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:3.0:1773941553.791293:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090851328 : -113743618700288 : ffff988d00030c00) 00000400:00000200:2.0:1773941553.791297:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941553.791307:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[88, 89] 00010000:00010000:3.0:1773941553.791310:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e29d0 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941553.791316:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:2.0:1773941553.791330:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [200:200:256:4294967295] 192.168.204.114@tcp LPNI seq info [90:90:8:4294967295] 00010000:00000001:3.0:1773941553.791352:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000200:2.0:1773941553.791354:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:3.0:1773941553.791369:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e29d0 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941553.791385:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00010000:00000001:3.0:1773941553.791401:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000800:00000200:2.0:1773941553.791412:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00010000:00010000:3.0:1773941553.791414:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e29d0 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000010:2.0:1773941553.791425:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0dbb00. 00010000:00000001:3.0:1773941553.791439:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000200:2.0:1773941553.791440:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:2.0:1773941553.791451:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00010000:00000001:3.0:1773941553.791460:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000200:2.0:1773941553.791461:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0dbb00 type 1, nob 616 niov 1 nkiov 1 00010000:00010000:3.0:1773941553.791472:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e29d0 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941553.791487:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941553.791490:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941553.791498:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941553.791499:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941553.791509:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294945387 00010000:00000001:3.0:1773941553.791513:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000100:00000001:2.0:1773941553.791519:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294945383 00010000:00010000:3.0:1773941553.791521:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e29d0 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941553.791531:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294945387 00010000:00010000:3.0:1773941553.791538:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d00030c00 already unused, calling callback (ffffffffc12b4c20) 00000100:00000001:2.0:1773941553.791542:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294945383 10000000:00000001:3.0:1773941553.791545:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000001:2.0:1773941553.791550:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 10000000:00010000:3.0:1773941553.791552:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e29d0 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941553.791560:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941553.791570:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941553.791572:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000100:00000001:2.0:1773941553.791578:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:3.0:1773941553.791579:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:3.0:1773941553.791584:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941553.791589:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030c00 refcount=5 00000020:00000001:3.0:1773941553.791592:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090851328 : -113743618700288 : ffff988d00030c00) 00000100:00000001:2.0:1773941553.791593:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:3.0:1773941553.791597:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090851328 : -113743618700288 : ffff988d00030c00) 00000100:00000001:2.0:1773941553.791599:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00010000:00000001:3.0:1773941553.791605:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:3.0:1773941553.791610:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e29d0 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941553.791630:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 10000000:00000001:3.0:1773941553.791635:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000001:2.0:1773941553.791636:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941553.791644:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 10000000:00010000:3.0:1773941553.791645:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e29d0 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941553.791650:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:01000000:3.0:1773941553.791668:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:3.0:1773941553.791675:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:3.0:1773941553.791679:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941553.791684:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00010000:3.0:1773941553.791686:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:3.0:1773941553.791696:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000800:00000010:1.0:1773941553.791698:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0dbb00. 00000400:00000200:1.0:1773941553.791711:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000001:3.0:1773941553.791716:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:3.0:1773941553.791722:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000400:00000200:1.0:1773941553.791730:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941553.791740:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516c38 00000400:00000010:1.0:1773941553.791746:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516c38. 00000100:00000001:1.0:1773941553.791752:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941553.791760:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d111f0a80 x1860112291487872/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941563 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941553.791773:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941553.791778:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d111f0a80 x1860112291487872/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941563 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941553.791787:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941553.791803:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:3.0:1773941553.791813:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:3.0:1773941553.791823:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1773941553.791830:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d00030c00 with handle 0x8e86767622e29d0 from hash 00010000:00000001:3.0:1773941553.791837:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:3.0:1773941553.791841:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941553.791844:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941553.791847:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:3.0:1773941553.791850:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:3.0:1773941553.791854:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:3.0:1773941553.791859:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941553.791861:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941553.791864:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941553.791868:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941553.791873:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e29d0 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:3.0:1773941553.791882:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941553.791884:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941553.791885:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:3.0:1773941553.791888:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:3.0:1773941553.791890:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:3.0:1773941553.791891:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941553.791893:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:3.0:1773941553.791896:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e29d0 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:3.0:1773941553.791905:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d21ec1800 count: 0 00010000:00000001:3.0:1773941553.791919:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:3.0:1773941553.791929:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941553.791932:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941553.791940:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10836680 x1860112291487744/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941569 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941553.791950:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941553.791953:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941553.792025:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d02ce7400. 02000000:00000001:3.0:1773941553.792030:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941553.792033:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941553.792036:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941553.792039:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941553.792042:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065c5e00. 02000000:00000001:3.0:1773941553.792047:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941553.792049:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941553.792052:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10836680. 00000100:00000001:3.0:1773941553.792056:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941553.792058:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:3.0:1773941553.792061:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:3.0:1773941553.792064:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:3.0:1773941553.792067:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:3.0:1773941553.792072:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d03ec7540. 00000020:00000010:3.0:1773941553.792078:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d1a1c8e00. 00000020:00000010:3.0:1773941553.792085:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d1a1c8c00. 00000040:00000010:3.0:1773941553.792090:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d1a1c8600. 00000020:00000010:3.0:1773941553.792096:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d08995000. 00000020:00000010:3.0:1773941553.792103:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d1a1c8200. 80000000:00000010:3.0:1773941553.792109:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d111f7c00. 00040000:00000010:3.0:1773941553.792115:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d1a1c9800. 20000000:00000010:3.0:1773941553.792120:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d1a1c8800. 10000000:00000040:3.0:1773941553.792126:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:3.0:1773941553.792129:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:3.0:1773941553.792135:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d1a1c8c00. 00000040:00000010:3.0:1773941553.792140:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d1a1c8600. 00000020:00000010:3.0:1773941553.792144:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d08995000. 00000020:00000010:3.0:1773941553.792149:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d1a1c8200. 80000000:00000010:3.0:1773941553.792152:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d111f7c00. 00040000:00000010:3.0:1773941553.792156:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d1a1c9800. 20000000:00000010:3.0:1773941553.792160:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d1a1c8800. 00000020:00000010:3.0:1773941553.792164:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d1a1c8e00. 10000000:00000010:3.0:1773941553.792167:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d03ec7540. 10000000:01000000:3.0:1773941553.792171:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:3.0:1773941553.792211:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:3.0:1773941553.792215:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:3.0:1773941553.792217:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:3.0:1773941553.792219:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:3.0:1773941553.792222:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:3.0:1773941553.792225:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941553.793646:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941553.793660:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941553.793664:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941553.793667:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941553.793674:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941553.793681:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941553.793690:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941553.793695:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941553.793699:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941553.793704:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324606080 00000400:00000200:0.0:1773941553.793709:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941553.793714:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xe51 [1] + 0 00000800:00000001:0.0:1773941553.793721:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941553.793730:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941553.793734:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941553.793738:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941553.793744:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941553.793751:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d111f0a80 x1860112291487872/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941563 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941553.793766:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d111f0a80 x1860112291487872/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941563 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941553.793805:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941553.793815:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941553.793819:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941553.793911:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941553.793917:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941553.793923:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941553.793925:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941553.793928:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941553.793935:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941553.793937:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941553.793941:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941553.793950:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d111f0a80 x1860112291487872/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941563 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941553.794011:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d111f0a80 x1860112291487872/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941563 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941553.794022:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941553.794025:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941553.794028:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941553.794031:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941553.794033:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941553.794036:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941553.794040:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941553.794046:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=54 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941553.794051:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941553.794055:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941553.794057:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941553.794060:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941553.794062:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941553.794066:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941553.794072:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d111f0a80 x1860112291487872/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941563 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941553.794081:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941553.794084:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941553.794086:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941553.794101:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941553.794107:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941553.794110:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941553.794112:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941553.794115:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941553.794118:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941553.794121:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941553.794124:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:2.0:1773941553.794128:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941553 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941553.794132:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941553.794140:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:2.0:1773941553.794142:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941553.794145:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941553.794147:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941553.794150:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941553.794154:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:2.0:1773941553.794157:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941553.794160:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941553.794165:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941553.794169:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:2.0:1773941553.794197:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941553.794508:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941553.794513:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941553.794524:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d111f0a80 x1860112291487872/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941563 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941553.794537:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d111f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291487872:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941553.794546:0:14762:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:3.0:1773941553.794547:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941553.794555:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941553.794560:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d111f0a80 x1860112291487872/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941563 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1773941553.794566:0:14762:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:2.0:1773941553.794577:0:14762:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941553.794584:0:14762:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:3.0:1773941553.794586:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 10000000:01000000:2.0:1773941553.794589:0:14762:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000001:3.0:1773941553.794592:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:2.0:1773941553.794596:0:14762:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000010:3.0:1773941553.794598:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a7400. 00000001:00000001:2.0:1773941553.794602:0:14762:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:3.0:1773941553.794606:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:2.0:1773941553.794609:0:14762:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941553.794612:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:2.0:1773941553.794616:0:14762:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000040:3.0:1773941553.794617:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000001:00000001:2.0:1773941553.794621:0:14762:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000001:3.0:1773941553.794625:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:2.0:1773941553.794627:0:14762:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941553.794631:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d03523000. 10000000:00000001:2.0:1773941553.794633:0:14762:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941553.794639:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1773941553.794640:0:14762:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000001:3.0:1773941553.794645:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941553.794649:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d111f0a80. 00000100:00080000:2.0:1773941553.794658:0:14762:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:3.0:1773941553.794661:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941553.794668:0:14762:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:3.0:1773941553.794670:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941553.794673:0:14762:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941553.794680:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941553.794681:0:14762:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1773941553.794693:0:14762:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941553.794696:0:14762:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:3.0:1773941553.794699:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 10000000:01000000:2.0:1773941553.794699:0:14762:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:2.0:1773941553.794709:0:14762:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00000001:3.0:1773941553.794710:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941553.794718:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941553.794722:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1773941553.794735:0:14762:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:2.0:1773941553.794739:0:14762:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941553.794742:0:14762:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:2.0:1773941553.794746:0:14762:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:2.0:1773941553.794750:0:14762:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941553.794754:0:14762:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941553.794756:0:14762:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941553.794759:0:14762:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941553.794761:0:14762:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:3.1:1773941553.798014:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d21ec1800. 00010000:00000010:3.1:1773941553.798028:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d00030c00. 00000020:00000001:0.0:1773941554.171117:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941554.171125:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941554.171131:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941555.067084:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294945387, at 4294945408 00000100:00000040:0.0:1773941555.067084:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294945387, at 4294945408 00000100:00000010:0.0:1773941555.067105:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580b40. 00000100:00000010:1.0:1773941555.067107:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580c60. 10000000:00000001:1.0:1773941555.131137:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941555.131144:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941555.131165:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941555.131168:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941555.131169:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941555.131171:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941555.131174:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941555.131176:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941555.131178:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:1.0:1773941555.131181:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941555.131182:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941555.131193:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c2680. 00000020:00000040:1.0:1773941555.131198:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941555.131202:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941555.131206:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941555.131211:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941555.131219:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d181fc200. 00000100:00000001:1.0:1773941555.131226:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941555.131230:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941555.131233:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941555.131241:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d04dd4f00. 00010000:00000001:1.0:1773941555.131247:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941555.131251:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d01489e00. 00000020:00000001:1.0:1773941555.131255:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941555.131257:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d01489e00 with handle 0x8e86767622e29d7 to hash 00000020:00000001:1.0:1773941555.131259:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941555.131261:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330112187904 : -113743597363712 : ffff988d01489e00) 00010000:00000001:1.0:1773941555.131263:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330112187904 : -113743597363712 : ffff988d01489e00) 00010000:00000001:1.0:1773941555.131267:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941555.131270:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941555.131273:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01489e00/0x8e86767622e29d7 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941555.131303:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d01489e00/0x8e86767622e29d7 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941555.131314:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d01489e00/0x8e86767622e29d7 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941555.131319:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941555.131321:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941555.131326:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941555.131328:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330528610752 : -113743180940864 : ffff988d1a1ab9c0) 00000100:00000001:1.0:1773941555.131331:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941555.131333:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631464576 : -113743078087040 : ffff988d203c2680) 00000100:00000001:1.0:1773941555.131337:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941555.131341:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941555.131343:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941555.131344:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941555.131354:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c2680 x1860112291488000/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941555.131363:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941555.131365:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941555.131368:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941555.131370:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941555.131372:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941555.131375:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c2680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291488000:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941555.131379:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941555.131384:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941555.131386:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941555.131388:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941555.131392:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d1118b400. 02000000:00000001:1.0:1773941555.131394:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941555.131401:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941555.131409:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c5d8. 00000400:00000010:1.0:1773941555.131415:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000100:00000200:1.0:1773941555.131421:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291488000, portal 25 00000100:00000001:1.0:1773941555.131423:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941555.131424:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631464576 : -113743078087040 : ffff988d203c2680) 00000100:00000040:1.0:1773941555.131429:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c2680 x1860112291488000/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941571 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941555.131435:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941555.131438:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941555.131441:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113dec38. 00000100:00000200:1.0:1773941555.131443:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291488000, offset 0 00000400:00000200:1.0:1773941555.131448:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941555.131456:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941555.131463:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941555.131468:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941555.131472:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941555.131476:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941555.131478:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941555.131481:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[90, 89] 00000400:00000200:1.0:1773941555.131484:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:1.0:1773941555.131487:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [201:201:256:4294967295] 192.168.204.124@tcp LPNI seq info [91:91:8:4294967295] 00000400:00000200:1.0:1773941555.131491:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941555.131500:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:1.0:1773941555.131505:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:1.0:1773941555.131509:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfdd00. 00000800:00000200:1.0:1773941555.131515:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:1.0:1773941555.131522:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:1.0:1773941555.131526:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfdd00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941555.131544:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941555.131548:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941555.131551:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294946727 00000100:00000010:1.0:1773941555.131559:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3360. 00000100:00000001:1.0:1773941555.131566:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1773941555.131572:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294946727 00000100:00000010:1.0:1773941555.131576:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3ba0. 00000100:00000001:1.0:1773941555.131579:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1773941555.131582:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941555.131584:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941555.131595:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941555.131597:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941555.131599:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 11 seconds 00000100:00000001:1.0:1773941555.131613:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941555.131616:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941555.131700:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941555.131705:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfdd00. 00000400:00000200:0.0:1773941555.131718:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941555.131724:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941555.131732:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113dec38 00000400:00000010:0.0:1773941555.131735:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113dec38. 00000100:00000001:0.0:1773941555.131739:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941555.131743:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c2680 x1860112291488000/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941571 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941555.131753:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941555.131756:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d203c2680 x1860112291488000/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941571 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941555.131763:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941555.131764:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941555.133341:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941555.133356:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941555.133360:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941555.133362:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941555.133367:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941555.133372:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941555.133378:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941555.133381:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941555.133385:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941555.133388:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324606100 00000400:00000200:0.0:1773941555.133399:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941555.133404:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xe59 [1] + 224 00000800:00000001:0.0:1773941555.133411:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941555.133431:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941555.133433:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941555.133437:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941555.133441:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941555.133446:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c2680 x1860112291488000/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941571 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941555.133455:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d203c2680 x1860112291488000/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941571 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941555.133474:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941555.133490:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941555.133495:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941555.133509:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941555.133517:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941555.133519:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941555.133522:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:1.0:1773941555.133527:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:1.0:1773941555.133529:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:1.0:1773941555.133532:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941555.133538:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c2680 x1860112291488000/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941571 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941555.133548:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c2680 x1860112291488000/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941571 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941555.133555:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941555.133556:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941555.133559:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941555.133561:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941555.133563:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941555.133566:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941555.133568:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941555.133584:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=56 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941555.133587:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941555.133590:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941555.133595:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:1.0:1773941555.133608:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941555.133611:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941555.133631:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941555.133634:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941555.133638:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941555.133642:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941555.133643:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941555.133645:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941555.133646:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941555.133649:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d203c2680 x1860112291488000/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941571 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941555.133655:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941555.133656:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941555.133658:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941555.133660:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941555.133663:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941555.133664:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941555.133666:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941555.133667:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941555.133668:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941555.133669:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941555.133670:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941555.133672:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941555.133673:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941555.133676:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941555.133678:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 276 00000400:00000200:1.0:1773941555.133681:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941555.133685:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941555.133689:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941555.133691:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941555.133693:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941555.133695:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941555.133702:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941555.133704:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941555.133705:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941555.133707:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941555.133710:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941555.133712:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941555.133716:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941555.133719:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941555.133721:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941555.133723:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941555.133727:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941555.133731:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941555.133734:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941555.133737:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941555.133741:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941555.133745:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941555.133748:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941555.133750:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941555.133755:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c3800. 00000020:00000040:1.0:1773941555.133758:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941555.133762:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941555.133765:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941555.133767:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941555.133769:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941555.133772:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941555.133778:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d11188c00. 00000100:00000001:1.0:1773941555.133783:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941555.133791:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c3800 x1860112291488128/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941555.133802:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941555.133805:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941555.133811:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000049debcf1] to pc [ptlrpcd_rcv+-1] req@ffff988d203c3800 x1860112291488128/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941555.133843:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941555.133845:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941555.133846:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941555.133849:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941555.133851:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941555.133852:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941555.133854:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941555.133856:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941555.133858:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941555.133862:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c2680 x1860112291488000/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941571 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941555.133868:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941555.133871:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941555.133872:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941555.133875:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941555.133878:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c2680 x1860112291488000/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941571 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941555.133885:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c2680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291488000:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941555.133900:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941555.133902:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941555.133904:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941555.133905:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941555.133908:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c2680 x1860112291488000/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941571 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941555.133913:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941555.133915:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941555.133919:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941555.133922:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941555.133923:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:1.0:1773941555.133928:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941555.133936:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941555.133938:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941555.133940:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01489e00 refcount=4 00000020:00000001:1.0:1773941555.133942:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112187904 : -113743597363712 : ffff988d01489e00) 00010000:00000001:1.0:1773941555.133948:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112187904 : -113743597363712 : ffff988d01489e00) 00010000:00010000:1.0:1773941555.133952:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d01489e00/0x8e86767622e29d7 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941555.134009:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00010000:00000001:1.0:1773941555.134010:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941555.134018:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00010000:00010000:1.0:1773941555.134020:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d01489e00/0x8e86767622e29d7 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941555.134028:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941555.134031:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01489e00/0x8e86767622e29d7 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:3.0:1773941555.134036:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c3800 x1860112291488128/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941555.134039:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941555.134042:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941555.134047:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d01489e00/0x8e86767622e29d7 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941555.134083:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941555.134089:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941555.134100:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941555.134104:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d01489e00/0x8e86767622e29d7 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941555.134108:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00010000:1.0:1773941555.134112:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d01489e00 already unused, calling callback (ffffffffc12b4c20) 00000100:00000001:3.0:1773941555.134115:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941555.134116:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941555.134121:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d01489e00/0x8e86767622e29d7 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941555.134129:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00010000:00000001:1.0:1773941555.134129:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 02000000:00000001:3.0:1773941555.134138:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941555.134139:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941555.134142:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941555.134145:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01489e00 refcount=5 00000020:00000001:1.0:1773941555.134163:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112187904 : -113743597363712 : ffff988d01489e00) 00010000:00000001:1.0:1773941555.134165:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112187904 : -113743597363712 : ffff988d01489e00) 02000000:00000001:3.0:1773941555.134170:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941555.134171:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941555.134178:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d01489e00/0x8e86767622e29d7 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1773941555.134185:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291488128:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941555.134230:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 10000000:00000001:1.0:1773941555.134238:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 02000000:00000001:3.0:1773941555.134240:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 10000000:00010000:1.0:1773941555.134243:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d01489e00/0x8e86767622e29d7 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941555.134245:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941555.134256:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 10000000:01000000:1.0:1773941555.134266:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 02000000:00000010:3.0:1773941555.134267:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03523000. 10000000:00000040:1.0:1773941555.134269:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 02000000:00000001:3.0:1773941555.134279:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941555.134279:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941555.134304:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941555.134307:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941555.134310:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000400:00000200:3.0:1773941555.134314:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941555.134319:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941555.134326:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941555.134328:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1773941555.134330:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00000020:00000040:1.0:1773941555.134333:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d01489e00 with handle 0x8e86767622e29d7 from hash 00010000:00000001:1.0:1773941555.134338:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941555.134340:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941555.134341:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941555.134342:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941555.134355:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941555.134356:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000400:00000010:3.0:1773941555.134358:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00010000:00000001:1.0:1773941555.134360:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941555.134365:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941555.134366:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941555.134368:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1773941555.134370:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291488128, portal 25 00010000:00010000:1.0:1773941555.134372:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d01489e00/0x8e86767622e29d7 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941555.134379:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941555.134385:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631469056 : -113743078082560 : ffff988d203c3800) 00000100:00000040:3.0:1773941555.134397:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c3800 x1860112291488128/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941565 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941555.134422:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941555.134430:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941555.134436:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000100:00000001:3.0:1773941555.134442:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:1.0:1773941555.134444:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941555.134447:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000040:3.0:1773941555.134451:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00000001:1.0:1773941555.134454:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000010:3.0:1773941555.134457:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac6e8. 00010000:00000001:1.0:1773941555.134457:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941555.134465:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d01489e00/0x8e86767622e29d7 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:3.0:1773941555.134467:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291488128, offset 0 00000400:00000200:3.0:1773941555.134485:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00000040:1.0:1773941555.134487:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d04dd4f00 count: 0 00000400:00000200:3.0:1773941555.134496:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941555.134509:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:1.0:1773941555.134510:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941555.134516:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000200:3.0:1773941555.134517:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941555.134524:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000100:00000001:1.0:1773941555.134525:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000400:00000200:3.0:1773941555.134539:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941555.134554:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000100:00000040:1.0:1773941555.134554:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c2680 x1860112291488000/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941571 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941555.134565:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[90, 91] 00000400:00000200:3.0:1773941555.134581:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941555.134591:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [202:202:256:4294967295] 192.168.204.114@tcp LPNI seq info [92:92:8:4294967295] 00000100:00000001:1.0:1773941555.134610:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000400:00000200:3.0:1773941555.134611:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 02000000:00000001:1.0:1773941555.134620:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941555.134628:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d1118b400. 00000400:00000200:3.0:1773941555.134635:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 02000000:00000001:1.0:1773941555.134638:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941555.134641:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941555.134648:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000800:00000200:3.0:1773941555.134649:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000020:00000001:1.0:1773941555.134664:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 00000800:00000010:3.0:1773941555.134666:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74f00. 02000000:00000010:1.0:1773941555.134668:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d181fc200. 02000000:00000001:1.0:1773941555.134674:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941555.134676:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000800:00000200:3.0:1773941555.134682:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000100:00000010:1.0:1773941555.134684:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c2680. 00000100:00000001:1.0:1773941555.134692:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941555.134694:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941555.134702:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000800:00000200:3.0:1773941555.134703:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 10000000:01000000:1.0:1773941555.134716:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000800:00000200:3.0:1773941555.134718:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74f00 type 1, nob 616 niov 1 nkiov 1 10000000:00000001:1.0:1773941555.134721:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941555.134729:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d04d67c80. 00000020:00000010:1.0:1773941555.134734:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d181fc200. 00000100:00000001:3.0:1773941555.134745:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941555.134745:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d181fd000. 00000040:00000010:1.0:1773941555.134749:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d181fce00. 00000020:00000010:1.0:1773941555.134754:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941555.134760:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d181fc800. 00000400:00000200:3.0:1773941555.134765:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 80000000:00000010:1.0:1773941555.134767:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d054f3240. 00000100:00000001:3.0:1773941555.134776:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294946730 00040000:00000010:1.0:1773941555.134776:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d181fd600. 20000000:00000010:1.0:1773941555.134781:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d181fd800. 10000000:00000040:1.0:1773941555.134787:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:1.0:1773941555.134790:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941555.134793:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294946727 00000020:00000010:1.0:1773941555.134795:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d181fd000. 00000040:00000010:1.0:1773941555.134803:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d181fce00. 00000100:00000001:3.0:1773941555.134809:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294946730 00000020:00000010:1.0:1773941555.134809:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941555.134814:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d181fc800. 80000000:00000010:1.0:1773941555.134819:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d054f3240. 00000100:00000001:3.0:1773941555.134823:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294946727 00040000:00000010:1.0:1773941555.134824:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d181fd600. 20000000:00000010:1.0:1773941555.134830:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d181fd800. 00000020:00000010:1.0:1773941555.134835:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d181fc200. 00000100:00000001:3.0:1773941555.134837:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 10000000:00000010:1.0:1773941555.134839:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d04d67c80. 10000000:01000000:1.0:1773941555.134846:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 00000100:00000001:3.0:1773941555.134848:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941555.134867:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941555.134868:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941555.134880:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941555.134894:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941555.134895:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941555.134899:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 00000100:00000001:3.0:1773941555.134900:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941555.134913:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000100:00000001:3.0:1773941555.134920:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000800:00000200:0.0:1773941555.134924:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1773941555.134925:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000800:00000010:0.0:1773941555.134935:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74f00. 00000100:00000001:3.0:1773941555.134936:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941555.134945:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000200:0.0:1773941555.134948:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1773941555.134949:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941555.134957:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941555.134984:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941555.134993:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac6e8 00000400:00000010:0.0:1773941555.134996:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac6e8. 00000100:00000001:0.0:1773941555.135004:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941555.135011:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c3800 x1860112291488128/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941565 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941555.135030:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941555.135034:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c3800 x1860112291488128/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941565 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941555.135041:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941555.135042:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941555.136781:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941555.136789:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941555.136792:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941555.136794:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941555.136812:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941555.136818:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941555.136825:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941555.136829:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941555.136833:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941555.136838:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324606180 00000400:00000200:0.0:1773941555.136844:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941555.136849:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xe69 [1] + 0 00000800:00000001:0.0:1773941555.136856:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941555.136866:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941555.136869:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941555.136873:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941555.136908:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941555.136916:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c3800 x1860112291488128/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941565 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941555.136931:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c3800 x1860112291488128/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941565 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941555.136955:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941555.137002:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941555.137028:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941555.137033:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941555.137040:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941555.137049:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941555.137052:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941555.137069:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941555.137077:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941555.137080:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941555.137084:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941555.137092:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c3800 x1860112291488128/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941565 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941555.137104:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c3800 x1860112291488128/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941565 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941555.137112:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941555.137113:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941555.137116:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941555.137119:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941555.137122:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941555.137124:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941555.137127:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941555.137134:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=56 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941555.137137:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941555.137140:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941555.137142:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941555.137145:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941555.137163:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941555.137166:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941555.137171:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c3800 x1860112291488128/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941565 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941555.137179:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941555.137182:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941555.137184:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941555.137187:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941555.137205:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941555.137207:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941555.137209:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941555.137212:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941555.137215:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941555.137217:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941555.137230:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941555.137235:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941555 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941555.137238:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941555.137246:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941555.137249:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941555.137252:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941555.137255:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941555.137256:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941555.137260:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941555.137263:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941555.137266:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941555.137270:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941555.137273:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941555.137276:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941555.137582:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941555.137589:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941555.137599:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c3800 x1860112291488128/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941565 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941555.137612:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291488128:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941555.137635:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941555.137640:0:14763:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941555.137640:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941555.137651:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c3800 x1860112291488128/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941565 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941555.137668:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941555.137671:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941555.137674:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03523000. 00000100:00080000:3.0:1773941555.137680:0:14763:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 02000000:00000001:2.0:1773941555.137680:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941555.137685:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941555.137691:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00100000:3.0:1773941555.137694:0:14763:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000020:00000001:2.0:1773941555.137696:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941555.137701:0:14763:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941555.137704:0:14763:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:3.0:1773941555.137710:0:14763:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941555.137714:0:14763:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941555.137721:0:14763:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941555.137724:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d11188c00. 00000001:00000001:3.0:1773941555.137727:0:14763:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:2.0:1773941555.137730:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000001:00000001:3.0:1773941555.137732:0:14763:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941555.137733:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941555.137743:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c3800. 00000001:00000001:3.0:1773941555.137744:0:14763:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941555.137748:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 10000000:00000001:3.0:1773941555.137754:0:14763:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941555.137755:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941555.137770:0:14763:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941555.137770:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941555.137783:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1773941555.137784:0:14763:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941555.137785:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941555.137794:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941555.137796:0:14763:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941555.137797:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941555.137820:0:14763:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941555.137825:0:14763:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1773941555.137830:0:14763:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:3.0:1773941555.137834:0:14763:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941555.137836:0:14763:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:3.0:1773941555.137849:0:14763:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:3.0:1773941555.137878:0:14763:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:3.0:1773941555.137882:0:14763:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941555.137886:0:14763:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:3.0:1773941555.137893:0:14763:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:3.0:1773941555.137928:0:14763:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941555.137932:0:14763:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941555.137936:0:14763:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941555.137940:0:14763:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941555.137943:0:14763:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941555.140044:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d04dd4f00. 00010000:00000010:1.1:1773941555.140053:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d01489e00. 00000020:00000001:2.0:1773941555.195208:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941555.195215:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941555.195220:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:0.0:1773941556.219186:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941556.219193:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941556.219199:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941556.411100:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294946730, at 4294946752 00000100:00000040:0.0:1773941556.411100:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294946730, at 4294946752 00000100:00000010:0.0:1773941556.411120:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3360. 00000100:00000010:1.0:1773941556.411143:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3ba0. 00000100:00000040:0.0:1773941556.475230:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) c1c1ff21-5f68-4bf6-bdc4-258644b24465->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1773941556.475248:0:8:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1773941556.475264:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10835f80. 00000020:00000040:0.0:1773941556.475275:0:8:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941556.475283:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941556.475289:0:8:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941556.475317:0:8:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941556.475332:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff988d0a20bd00. 00000100:00000001:0.0:1773941556.475344:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941556.475364:0:8:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging c1c1ff21-5f68-4bf6-bdc4-258644b24465->MGS req@ffff988d10835f80 x1860112291488256/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1773941556.475385:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941556.475393:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941556.475406:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000082649650] to pc [ptlrpcd_00_01+1] req@ffff988d10835f80 x1860112291488256/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941556.475466:0:8:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941556.475477:0:8:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (284) 00000100:00000001:1.0:1773941556.475503:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941556.475507:0:2401:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941556.475511:0:2402:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941556.475518:0:2401:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941556.475519:0:2402:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941556.475534:0:2402:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10835f80 x1860112291488256/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941556.475549:0:2402:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941556.475569:0:2402:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941556.475575:0:2402:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941556.475579:0:2402:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941556.475583:0:2402:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941556.475589:0:2402:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10835f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2402:1860112291488256:192.168.204.114@tcp:400:kworker.0 00000100:00000001:1.0:1773941556.475595:0:2402:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941556.475602:0:2402:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941556.475605:0:2402:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941556.475609:0:2402:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941556.475616:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff988d181fc200. 02000000:00000001:1.0:1773941556.475619:0:2402:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941556.475628:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941556.475634:0:2402:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c5d8. 00000400:00000010:1.0:1773941556.475643:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000100:00000200:1.0:1773941556.475651:0:2402:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112291488256, portal 25 00000100:00000001:1.0:1773941556.475654:0:2402:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941556.475657:0:2402:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367696768 : -113743341854848 : ffff988d10835f80) 00000100:00000040:1.0:1773941556.475665:0:2402:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10835f80 x1860112291488256/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941572 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941556.475675:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941556.475679:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941556.475682:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113decc0. 00000100:00000200:1.0:1773941556.475685:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1860112291488256, offset 0 00000400:00000200:1.0:1773941556.475694:0:2402:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941556.475704:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941556.475714:0:2402:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941556.475721:0:2402:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941556.475726:0:2402:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941556.475732:0:2402:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941556.475735:0:2402:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941556.475740:0:2402:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[92, 91] 00000400:00000200:1.0:1773941556.475745:0:2402:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:1.0:1773941556.475750:0:2402:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [203:203:256:4294967295] 192.168.204.124@tcp LPNI seq info [93:93:8:4294967295] 00000400:00000200:1.0:1773941556.475763:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941556.475777:0:2402:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:1.0:1773941556.475785:0:2402:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:1.0:1773941556.475792:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfd400. 00000800:00000200:1.0:1773941556.475801:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:1.0:1773941556.475810:0:2402:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:1.0:1773941556.475816:0:2402:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfd400 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1773941556.475839:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941556.475844:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941556.475848:0:2402:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294948071 00000100:00000010:1.0:1773941556.475853:0:2402:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3ba0. 00000100:00000001:1.0:1773941556.475862:0:2402:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1773941556.475870:0:2402:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294948071 00000100:00000010:1.0:1773941556.475875:0:2402:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3240. 00000100:00000001:1.0:1773941556.475880:0:2402:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1773941556.475885:0:2402:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941556.475889:0:2402:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941556.475893:0:2402:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941556.475897:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941556.475906:0:2402:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941556.475909:0:2402:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:1.0:1773941556.475913:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941556.475915:0:2402:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941556.475917:0:2402:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941556.475920:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941556.476092:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941556.476100:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfd400. 00000400:00000200:0.0:1773941556.476107:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941556.476115:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941556.476146:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113decc0 00000400:00000010:0.0:1773941556.476152:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113decc0. 00000100:00000001:0.0:1773941556.476159:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941556.476166:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10835f80 x1860112291488256/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941572 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941556.476179:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941556.476184:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10835f80 x1860112291488256/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941572 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941556.476193:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941556.476196:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941556.478616:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941556.478636:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941556.478641:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941556.478646:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941556.478656:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941556.478666:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941556.478678:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941556.478696:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941556.478704:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941556.478712:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324606200 00000400:00000200:0.0:1773941556.478721:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941556.478730:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xe71 [1] + 224 00000800:00000001:0.0:1773941556.478742:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941556.478757:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941556.478762:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941556.478769:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941556.478779:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941556.478790:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10835f80 x1860112291488256/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941572 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941556.478812:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff988d10835f80 x1860112291488256/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941572 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941556.478846:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941556.478861:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941556.478869:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941556.478952:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941556.478957:0:2402:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941556.479000:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941556.479004:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941556.479010:0:2402:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:1.0:1773941556.479019:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:1.0:1773941556.479023:0:2402:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:1.0:1773941556.479030:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941556.479057:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10835f80 x1860112291488256/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941572 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941556.479080:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10835f80 x1860112291488256/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941572 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941556.479096:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941556.479100:0:2402:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941556.479105:0:2402:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941556.479109:0:2402:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941556.479114:0:2402:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941556.479119:0:2402:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941556.479150:0:2402:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941556.479160:0:2402:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=57 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941556.479165:0:2402:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941556.479169:0:2402:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1773941556.479172:0:2402:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941556.479177:0:2402:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941556.479181:0:2402:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941556.479187:0:2402:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941556.479192:0:2402:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941556.479197:0:2402:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941556.479201:0:2402:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941556.479203:0:2402:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941556.479206:0:2402:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941556.479211:0:2402:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d10835f80 x1860112291488256/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941572 ref 1 fl Rpc:RNQU/200/0 rc 0/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941556.479221:0:2402:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941556.479223:0:2402:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941556.479225:0:2402:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941556.479230:0:2402:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941556.479233:0:2402:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941556.479235:0:2402:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941556.479237:0:2402:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941556.479239:0:2402:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941556.479250:0:2402:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941556.479252:0:2402:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941556.479254:0:2402:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941556.479258:0:2402:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941556.479261:0:2402:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941556.479265:0:2402:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941556.479269:0:2402:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 278 00000400:00000200:1.0:1773941556.479275:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941556.479281:0:2402:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941556.479287:0:2402:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941556.479292:0:2402:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941556.479295:0:2402:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941556.479298:0:2402:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941556.479303:0:2402:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941556.479305:0:2402:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941556.479308:0:2402:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941556.479312:0:2402:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941556.479315:0:2402:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941556.479318:0:2402:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941556.479322:0:2402:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941556.479325:0:2402:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941556.479327:0:2402:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941556.479330:0:2402:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941556.479333:0:2402:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941556.479337:0:2402:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941556.479340:0:2402:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941556.479344:0:2402:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941556.479348:0:2402:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941556.479351:0:2402:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941556.479354:0:2402:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941556.479356:0:2402:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941556.479362:0:2402:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c2680. 00000020:00000040:1.0:1773941556.479371:0:2402:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941556.479375:0:2402:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941556.479378:0:2402:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941556.479381:0:2402:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941556.479384:0:2402:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941556.479388:0:2402:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941556.479394:0:2402:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d1118b800. 00000100:00000001:1.0:1773941556.479399:0:2402:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941556.479408:0:2402:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c2680 x1860112291488384/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941556.479418:0:2402:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941556.479422:0:2402:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941556.479427:0:2402:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000097d1012f] to pc [ptlrpcd_rcv+-1] req@ffff988d203c2680 x1860112291488384/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941556.479458:0:2402:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941556.479462:0:2402:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941556.479464:0:2402:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:1.0:1773941556.479471:0:2402:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10835f80 x1860112291488256/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941572 ref 1 fl Rpc:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941556.479480:0:2402:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941556.479484:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941556.479487:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941556.479493:0:2402:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10835f80 x1860112291488256/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941572 ref 1 fl Interpret:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941556.479503:0:2402:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10835f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2402:1860112291488256:192.168.204.114@tcp:400:kworker.0 00000100:00000001:1.0:1773941556.479508:0:2402:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941556.479511:0:2402:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941556.479514:0:2402:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10835f80 x1860112291488256/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941572 ref 1 fl Complete:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941556.479523:0:2402:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941556.479533:0:2402:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941556.479536:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff988d181fc200. 02000000:00000001:1.0:1773941556.479541:0:2402:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941556.479543:0:2402:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941556.479547:0:2402:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941556.479550:0:2402:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941556.479554:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff988d0a20bd00. 02000000:00000001:1.0:1773941556.479559:0:2402:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941556.479561:0:2402:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941556.479564:0:2402:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10835f80. 00000100:00000001:1.0:1773941556.479569:0:2402:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941556.479572:0:2402:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941556.479576:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941556.479581:0:2402:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941556.479584:0:2402:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941556.479586:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941556.479589:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941556.479615:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941556.479632:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941556.479638:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941556.479661:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c2680 x1860112291488384/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941556.479685:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941556.479689:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941556.479697:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941556.479703:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941556.479709:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941556.479717:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291488384:192.168.204.114@tcp:250:ptlrpcd_00_01.0 00000100:00000001:2.0:1773941556.479726:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941556.479734:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941556.479739:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941556.479744:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941556.479759:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a4000. 02000000:00000001:2.0:1773941556.479763:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941556.479774:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941556.479781:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000400:00000010:2.0:1773941556.479791:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000200:2.0:1773941556.479801:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291488384, portal 25 00000100:00000001:2.0:1773941556.479804:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941556.479831:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631464576 : -113743078087040 : ffff988d203c2680) 00000100:00000040:2.0:1773941556.479841:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c2680 x1860112291488384/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941566 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941556.479852:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941556.479867:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:2.0:1773941556.479871:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516ee0. 00000100:00000200:2.0:1773941556.479877:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291488384, offset 0 00000400:00000200:2.0:1773941556.479886:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941556.479898:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941556.479909:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941556.479916:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941556.479922:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941556.479927:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941556.479931:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941556.479935:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[92, 93] 00000400:00000200:2.0:1773941556.479941:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:2.0:1773941556.479946:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [204:204:256:4294967295] 192.168.204.114@tcp LPNI seq info [94:94:8:4294967295] 00000400:00000200:2.0:1773941556.479952:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941556.480004:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:2.0:1773941556.480013:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:2.0:1773941556.480022:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0dbb00. 00000800:00000200:2.0:1773941556.480032:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:2.0:1773941556.480043:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:2.0:1773941556.480050:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0dbb00 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:2.0:1773941556.480077:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941556.480083:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941556.480087:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294948076 00000100:00000001:2.0:1773941556.480099:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294948071 00000100:00000001:2.0:1773941556.480106:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294948076 00000100:00000001:2.0:1773941556.480111:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294948071 00000100:00000001:2.0:1773941556.480159:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941556.480164:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941556.480169:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941556.480174:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941556.480185:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941556.480189:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941556.480193:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941556.480195:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941556.480197:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941556.480200:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941556.480293:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941556.480317:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0dbb00. 00000400:00000200:0.0:1773941556.480326:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941556.480344:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941556.480352:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516ee0 00000400:00000010:0.0:1773941556.480356:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516ee0. 00000100:00000001:0.0:1773941556.480362:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941556.480372:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c2680 x1860112291488384/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941566 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941556.480387:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941556.480394:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c2680 x1860112291488384/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941566 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941556.480406:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941556.480410:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941556.482946:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941556.483005:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941556.483010:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941556.483014:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941556.483024:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941556.483032:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941556.483043:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941556.483050:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941556.483058:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941556.483065:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324606280 00000400:00000200:0.0:1773941556.483074:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941556.483083:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xe81 [1] + 0 00000800:00000001:0.0:1773941556.483091:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941556.483104:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941556.483108:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941556.483115:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941556.483149:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941556.483158:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c2680 x1860112291488384/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941566 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941556.483178:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c2680 x1860112291488384/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941566 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941556.483211:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941556.483221:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941556.483227:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941556.483292:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941556.483300:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941556.483310:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941556.483314:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941556.483319:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941556.483331:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941556.483336:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941556.483343:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941556.483357:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c2680 x1860112291488384/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941566 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941556.483381:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c2680 x1860112291488384/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941566 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941556.483397:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941556.483401:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941556.483406:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941556.483411:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941556.483416:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941556.483420:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941556.483426:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941556.483437:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=57 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941556.483445:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941556.483451:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941556.483455:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941556.483462:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941556.483467:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941556.483472:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941556.483500:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c2680 x1860112291488384/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941566 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941556.483516:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941556.483521:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941556.483526:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941556.483531:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941556.483542:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941556.483548:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941556.483552:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941556.483557:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941556.483563:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941556.483569:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941556.483574:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:2.0:1773941556.483582:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941556 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941556.483589:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941556.483602:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:2.0:1773941556.483608:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941556.483614:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941556.483619:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941556.483623:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941556.483630:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:2.0:1773941556.483635:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941556.483641:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941556.483648:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941556.483655:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:2.0:1773941556.483661:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941556.484200:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941556.484208:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941556.484221:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c2680 x1860112291488384/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941566 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941556.484236:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291488384:192.168.204.114@tcp:250:ptlrpcd_00_01.0 00000100:00000001:3.0:1773941556.484247:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941556.484251:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941556.484255:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c2680 x1860112291488384/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941566 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941556.484265:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1773941556.484268:0:14765:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 02000000:00000001:3.0:1773941556.484269:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941556.484278:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a4000. 02000000:00000001:3.0:1773941556.484287:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941556.484290:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941556.484293:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941556.484302:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00080000:2.0:1773941556.484302:0:14765:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 02000000:00000010:3.0:1773941556.484313:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d1118b800. 00000100:00100000:2.0:1773941556.484327:0:14765:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000001:3.0:1773941556.484328:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941556.484336:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941556.484341:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c2680. 00000100:00000001:2.0:1773941556.484341:0:14765:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941556.484353:0:14765:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000001:3.0:1773941556.484356:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941556.484359:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941556.484366:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:2.0:1773941556.484368:0:14765:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:3.0:1773941556.484387:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941556.484391:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:2.0:1773941556.484392:0:14765:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:3.0:1773941556.484395:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941556.484400:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:2.0:1773941556.484406:0:14765:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:2.0:1773941556.484415:0:14765:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941556.484418:0:14765:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941556.484421:0:14765:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941556.484427:0:14765:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941556.484431:0:14765:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941556.484440:0:14765:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941556.484445:0:14765:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941556.484450:0:14765:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:2.0:1773941556.484455:0:14765:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1773941556.484461:0:14765:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941556.484465:0:14765:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941556.484469:0:14765:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:2.0:1773941556.484473:0:14765:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:2.0:1773941556.484511:0:14765:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:2.0:1773941556.484515:0:14765:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941556.484519:0:14765:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:2.0:1773941556.484529:0:14765:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:2.0:1773941556.484541:0:14765:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941556.484546:0:14765:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941556.484550:0:14765:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941556.484557:0:14765:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941556.484561:0:14765:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941556.603200:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941556.603209:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941556.603213:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941556.603216:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941556.603218:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941556.603219:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941556.603223:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941556.603243:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941556.603246:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:1.0:1773941556.603250:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941556.603253:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941556.603267:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c2680. 00000020:00000040:1.0:1773941556.603274:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941556.603281:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941556.603285:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941556.603293:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941556.603305:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d181fc200. 00000100:00000001:1.0:1773941556.603314:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941556.603320:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941556.603324:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941556.603334:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d04dd4f00. 00010000:00000001:1.0:1773941556.603362:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941556.603367:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d01488a00. 00000020:00000001:1.0:1773941556.603372:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941556.603376:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d01488a00 with handle 0x8e86767622e29de to hash 00000020:00000001:1.0:1773941556.603379:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941556.603381:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330112182784 : -113743597368832 : ffff988d01488a00) 00010000:00000001:1.0:1773941556.603385:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330112182784 : -113743597368832 : ffff988d01488a00) 00010000:00000001:1.0:1773941556.603391:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941556.603396:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941556.603401:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29de lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941556.603415:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29de lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941556.603432:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29de lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941556.603439:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941556.603442:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941556.603451:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941556.603454:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330528610752 : -113743180940864 : ffff988d1a1ab9c0) 00000100:00000001:1.0:1773941556.603457:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941556.603460:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631464576 : -113743078087040 : ffff988d203c2680) 00000100:00000001:1.0:1773941556.603466:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941556.603472:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941556.603475:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941556.603477:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941556.603491:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c2680 x1860112291488512/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941556.603504:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941556.603506:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941556.603511:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941556.603525:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941556.603528:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941556.603534:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c2680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291488512:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941556.603539:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941556.603546:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941556.603549:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941556.603552:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941556.603558:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d11188400. 02000000:00000001:1.0:1773941556.603561:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941556.603587:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941556.603592:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c5d8. 00000400:00000010:1.0:1773941556.603599:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000100:00000200:1.0:1773941556.603607:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291488512, portal 25 00000100:00000001:1.0:1773941556.603609:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941556.603610:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631464576 : -113743078087040 : ffff988d203c2680) 00000100:00000040:1.0:1773941556.603616:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c2680 x1860112291488512/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941572 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941556.603624:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941556.603627:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941556.603629:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de330. 00000100:00000200:1.0:1773941556.603633:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291488512, offset 0 00000400:00000200:1.0:1773941556.603639:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941556.603647:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941556.603656:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941556.603661:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941556.603665:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941556.603670:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941556.603672:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941556.603676:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[94, 93] 00000400:00000200:1.0:1773941556.603680:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:1.0:1773941556.603684:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [205:205:256:4294967295] 192.168.204.124@tcp LPNI seq info [95:95:8:4294967295] 00000400:00000200:1.0:1773941556.603689:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941556.603705:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:1.0:1773941556.603720:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:1.0:1773941556.603725:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfd400. 00000800:00000200:1.0:1773941556.603733:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:1.0:1773941556.603741:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:1.0:1773941556.603746:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfd400 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941556.603769:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941556.603774:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941556.603777:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294948199 00000100:00000001:1.0:1773941556.603786:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294948076 00000100:00000001:1.0:1773941556.603791:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294948199 00000100:00000001:1.0:1773941556.603795:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294948076 00000100:00000001:1.0:1773941556.603798:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941556.603801:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941556.603804:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941556.603806:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941556.603809:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 11 seconds 00000100:00000001:1.0:1773941556.603813:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941556.603816:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941556.603947:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941556.603954:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfd400. 00000400:00000200:0.0:1773941556.603990:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941556.603997:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941556.604003:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de330 00000400:00000010:0.0:1773941556.604005:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de330. 00000100:00000001:0.0:1773941556.604010:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941556.604016:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c2680 x1860112291488512/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941572 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941556.604044:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941556.604048:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d203c2680 x1860112291488512/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941572 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941556.604055:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941556.604057:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941556.605729:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941556.605746:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941556.605749:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941556.605751:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941556.605757:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941556.605763:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941556.605770:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941556.605774:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941556.605779:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941556.605785:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324606300 00000400:00000200:0.0:1773941556.605790:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941556.605795:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xe89 [1] + 224 00000800:00000001:0.0:1773941556.605802:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941556.605811:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941556.605813:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941556.605818:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941556.605824:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941556.605832:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c2680 x1860112291488512/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941572 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941556.605853:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d203c2680 x1860112291488512/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941572 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941556.605883:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941556.605897:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941556.605903:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941556.605951:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941556.605986:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941556.605989:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941556.605993:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:1.0:1773941556.606000:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:1.0:1773941556.606003:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:1.0:1773941556.606008:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941556.606016:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c2680 x1860112291488512/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941572 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941556.606054:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c2680 x1860112291488512/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941572 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941556.606071:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941556.606088:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941556.606094:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941556.606099:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941556.606104:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941556.606108:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941556.606113:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941556.606156:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=57 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941556.606181:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=0 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941556.606185:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941556.606193:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:1.0:1773941556.606197:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941556.606202:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941556.606207:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941556.606213:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941556.606219:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941556.606225:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941556.606245:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941556.606249:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941556.606252:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941556.606261:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d203c2680 x1860112291488512/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941572 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941556.606279:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941556.606282:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941556.606285:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941556.606290:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941556.606295:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941556.606299:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941556.606303:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941556.606305:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941556.606308:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941556.606312:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941556.606315:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941556.606319:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941556.606324:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941556.606345:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941556.606351:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 279 00000400:00000200:1.0:1773941556.606362:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941556.606370:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941556.606378:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941556.606384:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941556.606389:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941556.606394:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941556.606400:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941556.606404:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941556.606408:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941556.606412:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941556.606417:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941556.606422:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941556.606429:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941556.606433:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941556.606437:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941556.606441:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941556.606447:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941556.606454:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941556.606460:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941556.606465:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941556.606472:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941556.606477:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941556.606482:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941556.606486:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941556.606495:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c3800. 00000020:00000040:1.0:1773941556.606500:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941556.606506:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941556.606510:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941556.606514:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941556.606518:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941556.606535:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941556.606546:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d1118b800. 00000100:00000001:1.0:1773941556.606554:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941556.606568:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c3800 x1860112291488640/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941556.606598:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941556.606602:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941556.606609:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000049debcf1] to pc [ptlrpcd_rcv+-1] req@ffff988d203c3800 x1860112291488640/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941556.606642:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941556.606647:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941556.606650:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941556.606654:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941556.606658:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941556.606660:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941556.606664:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941556.606668:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941556.606672:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941556.606682:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c2680 x1860112291488512/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941572 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941556.606695:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941556.606704:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941556.606705:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941556.606709:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941556.606716:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941556.606717:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941556.606727:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941556.606735:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c2680 x1860112291488512/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941572 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941556.606748:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c3800 x1860112291488640/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941556.606800:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:1.0:1773941556.606803:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c2680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291488512:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941556.606805:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941556.606820:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1773941556.606823:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1773941556.606826:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941556.606836:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941556.606838:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00100000:3.0:1773941556.606844:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291488640:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941556.606854:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:3.0:1773941556.606857:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941556.606864:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941556.606873:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941556.606875:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1773941556.606878:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941556.606892:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03523800. 00000100:00000040:1.0:1773941556.606894:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c2680 x1860112291488512/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941572 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1773941556.606898:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941556.606921:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941556.606930:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1773941556.606933:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00000100:00000010:1.0:1773941556.606938:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1a1ab9c0. 00000400:00000010:3.0:1773941556.606945:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00000100:00000001:1.0:1773941556.606948:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941556.606953:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:1.0:1773941556.606958:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941556.607009:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941556.607013:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941556.607016:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488a00 refcount=4 00000020:00000001:1.0:1773941556.607020:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182784 : -113743597368832 : ffff988d01488a00) 00010000:00000001:1.0:1773941556.607028:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182784 : -113743597368832 : ffff988d01488a00) 00000100:00000200:3.0:1773941556.607032:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291488640, portal 25 00010000:00010000:1.0:1773941556.607037:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29de lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941556.607039:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941556.607045:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631469056 : -113743078082560 : ffff988d203c3800) 00000100:00000040:3.0:1773941556.607065:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c3800 x1860112291488640/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941566 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941556.607073:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941556.607094:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00010000:1.0:1773941556.607097:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29de lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:3.0:1773941556.607100:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:3.0:1773941556.607108:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acf68. 00000100:00000200:3.0:1773941556.607116:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291488640, offset 0 00010000:00000001:1.0:1773941556.607170:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000400:00000200:3.0:1773941556.607174:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00010000:1.0:1773941556.607183:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29de lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941556.607200:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941556.607207:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941556.607213:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29de lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941556.607229:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941556.607250:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941556.607255:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941556.607258:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:1.0:1773941556.607265:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:3.0:1773941556.607273:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941556.607285:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00010000:1.0:1773941556.607293:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29de lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941556.607302:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941556.607313:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941556.607329:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[94, 95] 00000400:00000200:3.0:1773941556.607357:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00010000:00010000:1.0:1773941556.607358:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d01488a00 already unused, calling callback (ffffffffc12b4c20) 00000400:00000200:3.0:1773941556.607369:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [206:206:256:4294967295] 192.168.204.114@tcp LPNI seq info [96:96:8:4294967295] 10000000:00000001:1.0:1773941556.607372:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:3.0:1773941556.607385:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 10000000:00010000:1.0:1773941556.607387:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29de lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941556.607416:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00010000:00000001:1.0:1773941556.607430:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000800:00000200:3.0:1773941556.607437:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00010000:00000001:1.0:1773941556.607440:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000800:00000010:3.0:1773941556.607448:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74c00. 00000020:00000001:1.0:1773941556.607449:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941556.607461:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488a00 refcount=5 00000800:00000200:3.0:1773941556.607473:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000020:00000001:1.0:1773941556.607474:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182784 : -113743597368832 : ffff988d01488a00) 00010000:00000001:1.0:1773941556.607488:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182784 : -113743597368832 : ffff988d01488a00) 00000800:00000200:3.0:1773941556.607500:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00010000:00000001:1.0:1773941556.607503:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941556.607514:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29de lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941556.607516:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74c00 type 1, nob 616 niov 1 nkiov 1 10000000:00000001:1.0:1773941556.607543:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941556.607550:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29de lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941556.607560:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 10000000:01000000:1.0:1773941556.607564:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941556.607575:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000400:00000200:3.0:1773941556.607578:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 10000000:00000001:1.0:1773941556.607583:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941556.607588:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294948203 00010000:00010000:1.0:1773941556.607617:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941556.607622:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000100:00000001:3.0:1773941556.607625:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294948199 00010000:00000001:1.0:1773941556.607628:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941556.607637:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000100:00000001:3.0:1773941556.607638:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294948203 00000100:00000001:3.0:1773941556.607652:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294948199 00010000:00000001:1.0:1773941556.607652:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941556.607662:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941556.607664:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941556.607675:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941556.607680:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d01488a00 with handle 0x8e86767622e29de from hash 00000100:00000001:3.0:1773941556.607681:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941556.607689:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:1.0:1773941556.607693:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941556.607700:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941556.607701:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941556.607705:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1773941556.607708:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941556.607711:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941556.607713:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941556.607720:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941556.607723:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000800:00000200:0.0:1773941556.607729:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00000001:1.0:1773941556.607730:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000100:00000001:3.0:1773941556.607731:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941556.607737:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941556.607748:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000010:0.0:1773941556.607749:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74c00. 00010000:00000001:1.0:1773941556.607754:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941556.607759:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941556.607759:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 10000000:00000001:1.0:1773941556.607769:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941556.607779:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29de lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941556.607779:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941556.607791:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acf68 00010000:00000001:1.0:1773941556.607796:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000010:0.0:1773941556.607798:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acf68. 00010000:00000001:1.0:1773941556.607802:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941556.607808:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000100:00000001:0.0:1773941556.607811:0:2393:0:(events.c:34:request_out_callback()) Process entered 00010000:00000001:1.0:1773941556.607812:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941556.607818:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941556.607820:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941556.607823:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000200:0.0:1773941556.607825:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c3800 x1860112291488640/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941566 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:1.0:1773941556.607828:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29de lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:1.0:1773941556.607859:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d04dd4f00 count: 0 00000100:00000001:0.0:1773941556.607863:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941556.607873:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c3800 x1860112291488640/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941566 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941556.607881:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941556.607887:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941556.607890:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941556.607894:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941556.607897:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:1.0:1773941556.607909:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c2680 x1860112291488512/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941572 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941556.607924:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941556.607928:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941556.607934:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d11188400. 02000000:00000001:1.0:1773941556.607940:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941556.607943:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941556.607946:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941556.607949:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941556.607952:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d181fc200. 02000000:00000001:1.0:1773941556.607956:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941556.607968:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941556.607971:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c2680. 00000100:00000001:1.0:1773941556.607977:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941556.607979:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941556.607983:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941556.607987:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941556.607990:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941556.608068:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d04d67c80. 00000020:00000010:1.0:1773941556.608077:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d181fc200. 00000020:00000010:1.0:1773941556.608087:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d181fd800. 00000040:00000010:1.0:1773941556.608096:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d181fd600. 00000020:00000010:1.0:1773941556.608105:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941556.608117:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d181fc800. 80000000:00000010:1.0:1773941556.608156:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d054f32a0. 00040000:00000010:1.0:1773941556.608182:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d181fce00. 20000000:00000010:1.0:1773941556.608188:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d181fd000. 10000000:00000040:1.0:1773941556.608195:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:1.0:1773941556.608199:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941556.608206:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d181fd800. 00000040:00000010:1.0:1773941556.608212:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d181fd600. 00000020:00000010:1.0:1773941556.608216:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941556.608223:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d181fc800. 80000000:00000010:1.0:1773941556.608245:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d054f32a0. 00040000:00000010:1.0:1773941556.608252:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d181fce00. 20000000:00000010:1.0:1773941556.608258:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d181fd000. 00000020:00000010:1.0:1773941556.608264:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d181fc200. 10000000:00000010:1.0:1773941556.608268:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d04d67c80. 10000000:01000000:1.0:1773941556.608273:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941556.608279:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941556.608283:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941556.608287:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941556.608290:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941556.608294:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941556.608299:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941556.610195:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941556.610207:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941556.610210:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941556.610212:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941556.610239:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941556.610246:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941556.610255:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941556.610260:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941556.610265:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941556.610269:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324606380 00000400:00000200:0.0:1773941556.610275:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941556.610281:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xe99 [1] + 0 00000800:00000001:0.0:1773941556.610289:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941556.610300:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941556.610304:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941556.610310:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941556.610317:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941556.610324:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c3800 x1860112291488640/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941566 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941556.610339:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c3800 x1860112291488640/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941566 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941556.610380:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941556.610390:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941556.610396:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941556.610439:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941556.610445:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941556.610454:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941556.610456:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941556.610461:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941556.610469:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941556.610472:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941556.610477:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941556.610492:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c3800 x1860112291488640/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941566 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941556.610515:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c3800 x1860112291488640/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941566 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941556.610531:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941556.610535:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941556.610540:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941556.610544:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941556.610549:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941556.610553:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941556.610558:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941556.610570:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=57 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941556.610589:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=0 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941556.610595:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941556.610598:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941556.610603:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941556.610607:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941556.610613:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941556.610622:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c3800 x1860112291488640/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941566 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941556.610633:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941556.610636:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941556.610639:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941556.610642:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941556.610651:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941556.610654:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941556.610671:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941556.610675:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941556.610679:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941556.610682:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941556.610686:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941556.610692:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941556 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941556.610697:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941556.610707:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941556.610710:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941556.610714:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941556.610717:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941556.610719:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941556.610724:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941556.610728:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941556.610732:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941556.610737:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941556.610741:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941556.610745:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941556.611180:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941556.611190:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941556.611204:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c3800 x1860112291488640/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941566 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941556.611235:0:14766:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:2.0:1773941556.611235:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291488640:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941556.611255:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1773941556.611263:0:14766:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00000001:2.0:1773941556.611264:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00100000:3.0:1773941556.611276:0:14766:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000040:2.0:1773941556.611281:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c3800 x1860112291488640/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941566 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941556.611284:0:14766:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941556.611290:0:14766:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:3.0:1773941556.611299:0:14766:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941556.611306:0:14766:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941556.611313:0:14766:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941556.611316:0:14766:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941556.611318:0:14766:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941556.611320:0:14766:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941556.611322:0:14766:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941556.611330:0:14766:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941556.611339:0:14766:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:3.0:1773941556.611349:0:14766:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941556.611356:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941556.611362:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1773941556.611367:0:14766:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941556.611372:0:14766:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941556.611373:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03523800. 00000100:00080000:3.0:1773941556.611377:0:14766:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:3.0:1773941556.611385:0:14766:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941556.611390:0:14766:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 02000000:00000001:2.0:1773941556.611390:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 10000000:00000040:3.0:1773941556.611399:0:14766:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000020:00000001:2.0:1773941556.611401:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941556.611410:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941556.611415:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941556.611421:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d1118b800. 02000000:00000001:2.0:1773941556.611429:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941556.611432:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:3.0:1773941556.611433:0:14766:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:00000010:2.0:1773941556.611438:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c3800. 10000000:00000001:3.0:1773941556.611439:0:14766:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941556.611445:0:14766:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:3.0:1773941556.611455:0:14766:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:2.0:1773941556.611455:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941556.611466:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941556.611467:0:14766:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941556.611479:0:14766:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:2.0:1773941556.611483:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:3.0:1773941556.611485:0:14766:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941556.611498:0:14766:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941556.611500:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941556.611503:0:14766:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941556.611506:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941556.611514:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941556.611517:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941556.614094:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d04dd4f00. 00010000:00000010:1.1:1773941556.614111:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d01488a00. 00000020:00000001:2.0:1773941557.243241:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941557.243252:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941557.243260:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941557.883131:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294948203, at 4294948224 00000100:00000040:0.0:1773941557.883131:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294948203, at 4294948224 00000100:00000010:0.0:1773941557.883152:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3ba0. 00000100:00000010:1.0:1773941557.883154:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3240. 00000020:00000001:0.0:1773941558.267137:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1773941558.267144:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941558.267148:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941558.651093:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941558.651103:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941558.651108:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941558.651110:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941558.651112:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941558.651113:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941558.651117:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941558.651118:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941558.651121:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:1.0:1773941558.651124:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941558.651125:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941558.651139:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c2680. 00000020:00000040:1.0:1773941558.651144:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941558.651149:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941558.651153:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941558.651159:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941558.651168:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d181fc200. 00000100:00000001:1.0:1773941558.651175:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941558.651180:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941558.651183:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941558.651191:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d04dd4f00. 00010000:00000001:1.0:1773941558.651197:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941558.651200:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d01488a00. 00000020:00000001:1.0:1773941558.651205:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941558.651207:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d01488a00 with handle 0x8e86767622e29e5 to hash 00000020:00000001:1.0:1773941558.651209:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941558.651210:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330112182784 : -113743597368832 : ffff988d01488a00) 00010000:00000001:1.0:1773941558.651214:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330112182784 : -113743597368832 : ffff988d01488a00) 00010000:00000001:1.0:1773941558.651218:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941558.651221:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941558.651225:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29e5 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941558.651257:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29e5 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941558.651268:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29e5 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941558.651273:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941558.651275:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941558.651283:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941558.651284:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330528610752 : -113743180940864 : ffff988d1a1ab9c0) 00000100:00000001:1.0:1773941558.651287:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941558.651288:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631464576 : -113743078087040 : ffff988d203c2680) 00000100:00000001:1.0:1773941558.651294:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941558.651298:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941558.651300:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941558.651301:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941558.651312:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c2680 x1860112291488768/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941558.651319:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941558.651322:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941558.651325:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941558.651327:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941558.651329:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941558.651332:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c2680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291488768:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941558.651334:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941558.651340:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941558.651342:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941558.651344:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941558.651348:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d11188400. 02000000:00000001:1.0:1773941558.651350:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941558.651358:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941558.651365:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c5d8. 00000400:00000010:1.0:1773941558.651372:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000100:00000200:1.0:1773941558.651380:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291488768, portal 25 00000100:00000001:1.0:1773941558.651381:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941558.651382:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631464576 : -113743078087040 : ffff988d203c2680) 00000100:00000040:1.0:1773941558.651386:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c2680 x1860112291488768/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941574 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941558.651392:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941558.651395:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941558.651397:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113dea18. 00000100:00000200:1.0:1773941558.651399:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291488768, offset 0 00000400:00000200:1.0:1773941558.651405:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941558.651414:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941558.651422:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941558.651427:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941558.651431:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941558.651435:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941558.651437:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941558.651439:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[96, 95] 00000400:00000200:1.0:1773941558.651442:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:1.0:1773941558.651446:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [207:207:256:4294967295] 192.168.204.124@tcp LPNI seq info [97:97:8:4294967295] 00000400:00000200:1.0:1773941558.651450:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941558.651457:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:1.0:1773941558.651463:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:1.0:1773941558.651467:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfdd00. 00000800:00000200:1.0:1773941558.651474:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:1.0:1773941558.651482:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:1.0:1773941558.651486:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfdd00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941558.651504:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941558.651508:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941558.651511:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294950247 00000100:00000010:1.0:1773941558.651518:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3240. 00000100:00000001:1.0:1773941558.651526:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1773941558.651535:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294950247 00000100:00000010:1.0:1773941558.651538:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f32a0. 00000100:00000001:1.0:1773941558.651541:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1773941558.651544:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941558.651546:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941558.651548:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941558.651550:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941558.651552:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 11 seconds 00000100:00000001:1.0:1773941558.651555:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941558.651558:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941558.651622:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941558.651628:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfdd00. 00000400:00000200:0.0:1773941558.651634:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941558.651639:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941558.651644:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113dea18 00000400:00000010:0.0:1773941558.651647:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113dea18. 00000100:00000001:0.0:1773941558.651650:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941558.651655:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c2680 x1860112291488768/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941574 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941558.651664:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941558.651667:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d203c2680 x1860112291488768/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941574 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941558.651673:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941558.651675:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941558.652940:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941558.652955:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941558.652958:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941558.652985:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941558.652990:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941558.652994:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941558.652999:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941558.653002:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941558.653007:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941558.653010:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324606400 00000400:00000200:0.0:1773941558.653020:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941558.653024:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xea1 [1] + 224 00000800:00000001:0.0:1773941558.653031:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941558.653039:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941558.653041:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941558.653044:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941558.653048:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941558.653052:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c2680 x1860112291488768/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941574 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941558.653060:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d203c2680 x1860112291488768/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941574 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941558.653074:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941558.653092:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941558.653096:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941558.653114:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941558.653121:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941558.653122:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941558.653125:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:1.0:1773941558.653128:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:1.0:1773941558.653129:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:1.0:1773941558.653131:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941558.653136:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c2680 x1860112291488768/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941574 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941558.653144:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c2680 x1860112291488768/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941574 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941558.653149:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941558.653150:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941558.653152:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941558.653154:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941558.653156:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941558.653158:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941558.653160:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941558.653165:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=59 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941558.653168:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941558.653170:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941558.653174:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:1.0:1773941558.653182:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941558.653185:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941558.653187:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941558.653191:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941558.653194:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941558.653197:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941558.653199:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941558.653201:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941558.653203:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941558.653205:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d203c2680 x1860112291488768/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941574 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941558.653210:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941558.653211:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941558.653213:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941558.653215:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941558.653217:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941558.653218:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941558.653220:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941558.653220:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941558.653221:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941558.653222:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941558.653223:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941558.653225:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941558.653227:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941558.653229:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941558.653231:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 279 00000400:00000200:1.0:1773941558.653235:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941558.653238:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941558.653242:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941558.653245:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941558.653247:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941558.653249:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941558.653257:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941558.653259:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941558.653261:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941558.653263:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941558.653265:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941558.653267:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941558.653269:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941558.653271:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941558.653272:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941558.653273:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941558.653276:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941558.653278:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941558.653280:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941558.653282:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941558.653284:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941558.653286:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941558.653288:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941558.653289:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941558.653292:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c0000. 00000020:00000040:1.0:1773941558.653294:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941558.653296:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941558.653298:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941558.653301:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941558.653303:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941558.653305:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941558.653310:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d1118b400. 00000100:00000001:1.0:1773941558.653314:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941558.653320:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c0000 x1860112291488896/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941558.653329:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941558.653332:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941558.653337:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001ea6e639] to pc [ptlrpcd_rcv+-1] req@ffff988d203c0000 x1860112291488896/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941558.653367:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941558.653370:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941558.653372:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941558.653375:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941558.653378:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941558.653379:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941558.653381:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941558.653382:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941558.653384:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941558.653388:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c2680 x1860112291488768/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941574 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941558.653394:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941558.653396:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941558.653398:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941558.653400:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941558.653403:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c2680 x1860112291488768/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941574 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941558.653408:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c2680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291488768:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941558.653412:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941558.653415:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941558.653417:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941558.653420:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941558.653422:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c2680 x1860112291488768/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941574 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941558.653428:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941558.653430:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941558.653432:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:2.0:1773941558.653434:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941558.653434:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:1.0:1773941558.653443:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000001:2.0:1773941558.653444:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941558.653447:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00010000:00000001:1.0:1773941558.653453:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941558.653455:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941558.653458:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488a00 refcount=4 00000100:00000040:2.0:1773941558.653461:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c0000 x1860112291488896/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1773941558.653463:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182784 : -113743597368832 : ffff988d01488a00) 00010000:00000001:1.0:1773941558.653472:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182784 : -113743597368832 : ffff988d01488a00) 00010000:00010000:1.0:1773941558.653484:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29e5 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941558.653487:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941558.653493:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941558.653504:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00010000:00000001:1.0:1773941558.653505:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 02000000:00000001:2.0:1773941558.653510:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00010000:00010000:1.0:1773941558.653513:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29e5 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941558.653515:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941558.653528:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291488896:192.168.204.114@tcp:250:ll_cfg_requeue.0 00010000:00000001:1.0:1773941558.653543:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000001:2.0:1773941558.653544:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00010000:1.0:1773941558.653549:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29e5 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941558.653551:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00000001:1.0:1773941558.653556:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941558.653558:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941558.653559:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29e5 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941558.653563:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941558.653565:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941558.653569:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941558.653570:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 02000000:00000001:2.0:1773941558.653575:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00010000:1.0:1773941558.653576:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29e5 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000010:2.0:1773941558.653584:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a4800. 00010000:00010000:1.0:1773941558.653587:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d01488a00 already unused, calling callback (ffffffffc12b4c20) 02000000:00000001:2.0:1773941558.653588:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941558.653595:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:2.0:1773941558.653599:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 10000000:00010000:1.0:1773941558.653601:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29e5 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941558.653604:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000400:00000010:2.0:1773941558.653611:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00010000:00000001:1.0:1773941558.653617:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000100:00000200:2.0:1773941558.653620:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291488896, portal 25 00000100:00000001:2.0:1773941558.653622:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941558.653623:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631454720 : -113743078096896 : ffff988d203c0000) 00010000:00000001:1.0:1773941558.653629:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000040:2.0:1773941558.653633:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c0000 x1860112291488896/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941568 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1773941558.653634:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1773941558.653645:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941558.653647:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:2.0:1773941558.653649:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516d48. 00000020:00000040:1.0:1773941558.653651:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488a00 refcount=5 00000100:00000200:2.0:1773941558.653653:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291488896, offset 0 00000020:00000001:1.0:1773941558.653660:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182784 : -113743597368832 : ffff988d01488a00) 00000400:00000200:2.0:1773941558.653667:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00000001:1.0:1773941558.653667:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182784 : -113743597368832 : ffff988d01488a00) 00010000:00000001:1.0:1773941558.653675:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:2.0:1773941558.653680:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:1.0:1773941558.653680:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29e5 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941558.653689:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 10000000:00000001:1.0:1773941558.653689:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941558.653698:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29e5 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941558.653700:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941558.653710:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 10000000:01000000:1.0:1773941558.653717:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:2.0:1773941558.653722:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 10000000:00000040:1.0:1773941558.653729:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000400:00000200:2.0:1773941558.653733:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 10000000:00000001:1.0:1773941558.653733:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941558.653742:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000400:00000200:2.0:1773941558.653743:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[96, 97] 00010000:00000001:1.0:1773941558.653746:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:2.0:1773941558.653749:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00010000:00000001:1.0:1773941558.653750:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941558.653753:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941558.653758:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:2.0:1773941558.653759:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [208:208:256:4294967295] 192.168.204.114@tcp LPNI seq info [98:98:8:4294967295] 00010000:00000001:1.0:1773941558.653768:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941558.653777:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000020:00000040:1.0:1773941558.653778:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d01488a00 with handle 0x8e86767622e29e5 from hash 00010000:00000001:1.0:1773941558.653785:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941558.653787:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941558.653789:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941558.653792:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941558.653796:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000400:00000200:2.0:1773941558.653804:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00010000:00000001:1.0:1773941558.653805:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941558.653819:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941558.653822:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000200:2.0:1773941558.653827:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00010000:00000001:1.0:1773941558.653827:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941558.653838:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1773941558.653840:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0dbd00. 00010000:00010000:1.0:1773941558.653845:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29e5 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941558.653856:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:2.0:1773941558.653869:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00010000:00000001:1.0:1773941558.653873:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941558.653874:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941558.653877:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941558.653887:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000800:00000200:2.0:1773941558.653889:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0dbd00 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941558.653889:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941558.653893:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941558.653896:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941558.653899:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29e5 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:1.0:1773941558.653904:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d04dd4f00 count: 0 00000100:00000001:2.0:1773941558.653909:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941558.653915:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941558.653917:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941558.653918:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294950249 00010000:00000001:1.0:1773941558.653921:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941558.653926:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941558.653930:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294950247 00000100:00000040:1.0:1773941558.653933:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c2680 x1860112291488768/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941574 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941558.653948:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294950249 00000100:00000001:1.0:1773941558.653950:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941558.653957:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1773941558.654009:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294950247 00000800:00000200:0.0:1773941558.654012:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1773941558.654017:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000010:0.0:1773941558.654017:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0dbd00. 02000000:00000010:1.0:1773941558.654022:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d11188400. 00000400:00000200:0.0:1773941558.654023:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1773941558.654025:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941558.654027:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941558.654034:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000400:00000200:0.0:1773941558.654034:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:2.0:1773941558.654037:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941558.654038:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000400:00000200:0.0:1773941558.654039:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516d48 00000020:00000001:1.0:1773941558.654049:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 00000400:00000010:0.0:1773941558.654049:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516d48. 00000100:00000001:2.0:1773941558.654050:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:1.0:1773941558.654055:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d181fc200. 00000100:00000001:0.0:1773941558.654057:0:2393:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:1.0:1773941558.654064:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000200:0.0:1773941558.654065:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c0000 x1860112291488896/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941568 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1773941558.654066:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1773941558.654067:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000010:1.0:1773941558.654070:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c2680. 00000100:00000001:2.0:1773941558.654078:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1773941558.654093:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941558.654094:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1773941558.654100:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941558.654103:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941558.654104:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1773941558.654107:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c0000 x1860112291488896/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941568 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941558.654113:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 10000000:01000000:1.0:1773941558.654113:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941558.654121:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000100:00000001:2.0:1773941558.654122:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 10000000:00000010:1.0:1773941558.654127:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d04d67c80. 00000100:00000001:0.0:1773941558.654127:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941558.654129:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941558.654132:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:1.0:1773941558.654136:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d181fc200. 00000020:00000010:1.0:1773941558.654147:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d181fd000. 00000040:00000010:1.0:1773941558.654151:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d181fce00. 00000020:00000010:1.0:1773941558.654157:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941558.654161:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d181fc800. 80000000:00000010:1.0:1773941558.654165:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d054f3060. 00040000:00000010:1.0:1773941558.654169:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d181fd600. 20000000:00000010:1.0:1773941558.654173:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d181fd800. 10000000:00000040:1.0:1773941558.654178:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:1.0:1773941558.654179:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941558.654184:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d181fd000. 00000040:00000010:1.0:1773941558.654187:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d181fce00. 00000020:00000010:1.0:1773941558.654189:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941558.654192:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d181fc800. 80000000:00000010:1.0:1773941558.654195:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d054f3060. 00040000:00000010:1.0:1773941558.654199:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d181fd600. 20000000:00000010:1.0:1773941558.654201:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d181fd800. 00000020:00000010:1.0:1773941558.654203:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d181fc200. 10000000:00000010:1.0:1773941558.654205:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d04d67c80. 10000000:01000000:1.0:1773941558.654207:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941558.654209:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941558.654211:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941558.654213:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941558.654214:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941558.654215:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941558.654217:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941558.655561:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941558.655570:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941558.655573:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941558.655576:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941558.655589:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941558.655596:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941558.655602:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941558.655606:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941558.655610:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941558.655614:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324606480 00000400:00000200:0.0:1773941558.655618:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941558.655623:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xeb1 [1] + 0 00000800:00000001:0.0:1773941558.655629:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941558.655638:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941558.655640:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941558.655644:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941558.655648:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941558.655653:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c0000 x1860112291488896/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941568 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941558.655664:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c0000 x1860112291488896/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941568 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941558.655685:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941558.655691:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941558.655695:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941558.655720:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941558.655723:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941558.655728:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941558.655729:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941558.655731:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941558.655735:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941558.655737:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941558.655740:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941558.655746:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c0000 x1860112291488896/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941568 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941558.655754:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c0000 x1860112291488896/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941568 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941558.655759:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941558.655760:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941558.655762:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941558.655764:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941558.655766:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941558.655775:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941558.655777:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941558.655783:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=59 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941558.655785:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941558.655787:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941558.655788:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941558.655791:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941558.655792:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941558.655794:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941558.655797:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c0000 x1860112291488896/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941568 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941558.655802:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941558.655804:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941558.655806:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941558.655808:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941558.655813:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941558.655815:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941558.655817:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941558.655819:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941558.655821:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941558.655823:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941558.655824:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:2.0:1773941558.655827:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941558 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941558.655829:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941558.655834:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:2.0:1773941558.655836:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941558.655838:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941558.655840:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941558.655841:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941558.655844:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:2.0:1773941558.655845:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941558.655847:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941558.655850:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941558.655856:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:2.0:1773941558.655858:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941558.656137:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941558.656145:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941558.656153:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c0000 x1860112291488896/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941568 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941558.656163:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291488896:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941558.656166:0:14767:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:3.0:1773941558.656168:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941558.656170:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941558.656174:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c0000 x1860112291488896/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941568 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1773941558.656174:0:14767:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:2.0:1773941558.656178:0:14767:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941558.656180:0:14767:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:3.0:1773941558.656181:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 10000000:01000000:2.0:1773941558.656182:0:14767:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000001:3.0:1773941558.656184:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:2.0:1773941558.656185:0:14767:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000010:3.0:1773941558.656187:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a4800. 00000001:00000001:2.0:1773941558.656187:0:14767:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:3.0:1773941558.656190:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941558.656191:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:2.0:1773941558.656191:0:14767:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1773941558.656193:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000001:00000001:2.0:1773941558.656193:0:14767:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941558.656194:0:14767:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000001:3.0:1773941558.656196:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:2.0:1773941558.656196:0:14767:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941558.656198:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d1118b400. 10000000:00000001:2.0:1773941558.656198:0:14767:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941558.656199:0:14767:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000001:3.0:1773941558.656201:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00080000:2.0:1773941558.656203:0:14767:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941558.656206:0:14767:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941558.656208:0:14767:0:(recover.c:147:ptlrpc_resend()) Process entered 02000000:00000001:3.0:1773941558.656209:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1773941558.656210:0:14767:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941558.656211:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c0000. 00000100:00080000:2.0:1773941558.656213:0:14767:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:3.0:1773941558.656214:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941558.656215:0:14767:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:3.0:1773941558.656216:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:01000000:2.0:1773941558.656217:0:14767:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000100:00000001:3.0:1773941558.656218:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:00000040:2.0:1773941558.656219:0:14767:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00000001:3.0:1773941558.656222:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941558.656223:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941558.656226:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941558.656227:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1773941558.656236:0:14767:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:2.0:1773941558.656238:0:14767:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941558.656240:0:14767:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:2.0:1773941558.656244:0:14767:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:2.0:1773941558.656248:0:14767:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941558.656252:0:14767:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941558.656254:0:14767:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941558.656258:0:14767:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941558.656261:0:14767:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941558.659001:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d04dd4f00. 00010000:00000010:1.1:1773941558.659010:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d01488a00. 00000020:00000001:2.0:1773941559.291220:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941559.291232:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941559.291240:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941559.931158:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294950249, at 4294950272 00000100:00000040:0.0:1773941559.931158:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294950249, at 4294950272 00000100:00000010:0.0:1773941559.931180:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3240. 00000100:00000010:1.0:1773941559.931183:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f32a0. 10000000:00000001:1.0:1773941560.059117:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941560.059128:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941560.059134:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941560.059138:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941560.059139:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941560.059141:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941560.059145:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941560.059171:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941560.059173:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:1.0:1773941560.059177:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941560.059179:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941560.059188:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c2680. 00000020:00000040:1.0:1773941560.059199:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941560.059205:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941560.059209:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941560.059216:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941560.059225:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d181fc200. 00000100:00000001:1.0:1773941560.059234:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941560.059240:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941560.059244:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941560.059254:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d04dd4f00. 00010000:00000001:1.0:1773941560.059261:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941560.059266:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d01488a00. 00000020:00000001:1.0:1773941560.059272:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941560.059275:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d01488a00 with handle 0x8e86767622e29ec to hash 00000020:00000001:1.0:1773941560.059277:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941560.059279:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330112182784 : -113743597368832 : ffff988d01488a00) 00010000:00000001:1.0:1773941560.059283:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330112182784 : -113743597368832 : ffff988d01488a00) 00010000:00000001:1.0:1773941560.059289:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941560.059292:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941560.059297:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29ec lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941560.059310:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29ec lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941560.059323:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29ec lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941560.059330:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941560.059332:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941560.059350:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941560.059353:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330528610752 : -113743180940864 : ffff988d1a1ab9c0) 00000100:00000001:1.0:1773941560.059356:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941560.059358:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631464576 : -113743078087040 : ffff988d203c2680) 00000100:00000001:1.0:1773941560.059364:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941560.059369:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941560.059372:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941560.059375:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941560.059387:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c2680 x1860112291489024/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941560.059397:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941560.059400:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941560.059404:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941560.059408:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941560.059411:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941560.059415:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c2680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291489024:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941560.059419:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941560.059426:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941560.059429:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941560.059432:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941560.059438:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d11188c00. 02000000:00000001:1.0:1773941560.059440:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941560.059449:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941560.059454:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c5d8. 00000400:00000010:1.0:1773941560.059465:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000100:00000200:1.0:1773941560.059475:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291489024, portal 25 00000100:00000001:1.0:1773941560.059477:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941560.059478:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631464576 : -113743078087040 : ffff988d203c2680) 00000100:00000040:1.0:1773941560.059485:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c2680 x1860112291489024/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941576 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941560.059493:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941560.059497:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941560.059500:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de440. 00000100:00000200:1.0:1773941560.059509:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291489024, offset 0 00000400:00000200:1.0:1773941560.059517:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941560.059528:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941560.059537:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941560.059543:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941560.059548:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941560.059553:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941560.059556:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941560.059560:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[98, 97] 00000400:00000200:1.0:1773941560.059565:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:1.0:1773941560.059569:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [209:209:256:4294967295] 192.168.204.124@tcp LPNI seq info [99:99:8:4294967295] 00000400:00000200:1.0:1773941560.059574:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941560.059587:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:1.0:1773941560.059594:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:1.0:1773941560.059600:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfd300. 00000800:00000200:1.0:1773941560.059610:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:1.0:1773941560.059619:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:1.0:1773941560.059625:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfd300 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941560.059646:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941560.059650:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941560.059653:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294951655 00000100:00000010:1.0:1773941560.059659:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f32a0. 00000100:00000001:1.0:1773941560.059668:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1773941560.059677:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294951655 00000100:00000010:1.0:1773941560.059681:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3060. 00000100:00000001:1.0:1773941560.059686:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1773941560.059690:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941560.059693:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941560.059696:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941560.059698:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941560.059706:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 11 seconds 00000100:00000001:1.0:1773941560.059710:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941560.059714:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941560.059780:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941560.059786:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfd300. 00000400:00000200:0.0:1773941560.059791:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941560.059796:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941560.059803:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de440 00000400:00000010:0.0:1773941560.059805:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de440. 00000100:00000001:0.0:1773941560.059809:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941560.059814:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c2680 x1860112291489024/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941576 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941560.059832:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941560.059835:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d203c2680 x1860112291489024/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941576 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941560.059841:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941560.059842:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941560.061138:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941560.061153:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941560.061156:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941560.061159:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941560.061166:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941560.061172:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941560.061180:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941560.061184:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941560.061190:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941560.061195:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324606500 00000400:00000200:0.0:1773941560.061201:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941560.061208:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xeb9 [1] + 224 00000800:00000001:0.0:1773941560.061217:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941560.061227:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941560.061230:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941560.061234:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941560.061239:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941560.061244:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c2680 x1860112291489024/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941576 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941560.061256:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d203c2680 x1860112291489024/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941576 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941560.061277:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941560.061284:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941560.061288:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941560.061320:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941560.061327:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941560.061329:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941560.061332:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:1.0:1773941560.061336:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:1.0:1773941560.061339:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:1.0:1773941560.061342:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941560.061346:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c2680 x1860112291489024/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941576 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941560.061358:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c2680 x1860112291489024/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941576 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941560.061366:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941560.061368:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941560.061371:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941560.061374:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941560.061377:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941560.061380:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941560.061383:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941560.061390:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=61 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941560.061395:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941560.061399:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941560.061403:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:1.0:1773941560.061407:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941560.061410:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941560.061414:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941560.061419:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941560.061423:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941560.061427:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941560.061431:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941560.061438:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941560.061441:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941560.061446:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d203c2680 x1860112291489024/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941576 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941560.061455:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941560.061457:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941560.061459:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941560.061462:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941560.061465:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941560.061467:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941560.061469:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941560.061471:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941560.061472:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941560.061473:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941560.061475:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941560.061478:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941560.061481:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941560.061484:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941560.061487:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 281 00000400:00000200:1.0:1773941560.061492:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941560.061497:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941560.061502:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941560.061505:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941560.061509:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941560.061512:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941560.061515:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941560.061517:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941560.061520:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941560.061523:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941560.061526:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941560.061529:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941560.061532:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941560.061534:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941560.061536:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941560.061544:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941560.061547:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941560.061551:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941560.061554:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941560.061557:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941560.061560:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941560.061564:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941560.061566:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941560.061569:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941560.061573:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c2d80. 00000020:00000040:1.0:1773941560.061576:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941560.061580:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941560.061582:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941560.061585:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941560.061587:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941560.061589:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941560.061594:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d1118a400. 00000100:00000001:1.0:1773941560.061598:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941560.061605:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c2d80 x1860112291489152/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941560.061614:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941560.061617:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941560.061621:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c459535b] to pc [ptlrpcd_rcv+-1] req@ffff988d203c2d80 x1860112291489152/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941560.061645:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941560.061648:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941560.061650:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941560.061653:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941560.061655:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941560.061657:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941560.061659:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941560.061661:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941560.061664:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941560.061678:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c2680 x1860112291489024/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941576 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941560.061687:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941560.061690:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941560.061693:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941560.061695:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941560.061700:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c2680 x1860112291489024/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941576 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941560.061702:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941560.061712:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1773941560.061718:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c2680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291489024:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941560.061719:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941560.061730:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941560.061733:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c2d80 x1860112291489152/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941560.061735:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941560.061747:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941560.061750:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1773941560.061761:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1773941560.061762:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c2680 x1860112291489024/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941576 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941560.061768:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941560.061779:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1773941560.061782:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941560.061785:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000010:1.0:1773941560.061787:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1a1ab9c0. 02000000:00000001:3.0:1773941560.061791:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941560.061794:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941560.061799:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00100000:3.0:1773941560.061801:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291489152:192.168.204.114@tcp:250:ll_cfg_requeue.0 00010000:00000001:1.0:1773941560.061808:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000001:3.0:1773941560.061810:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00000001:1.0:1773941560.061813:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941560.061817:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:3.0:1773941560.061819:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000020:00000040:1.0:1773941560.061820:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488a00 refcount=4 02000000:00000001:3.0:1773941560.061826:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941560.061827:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182784 : -113743597368832 : ffff988d01488a00) 02000000:00000001:3.0:1773941560.061836:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:1.0:1773941560.061836:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182784 : -113743597368832 : ffff988d01488a00) 00010000:00010000:1.0:1773941560.061844:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29ec lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000010:3.0:1773941560.061845:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03522400. 02000000:00000001:3.0:1773941560.061854:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941560.061866:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000200:3.0:1773941560.061868:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941560.061876:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00010000:00010000:1.0:1773941560.061878:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29ec lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:3.0:1773941560.061886:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00000100:00000200:3.0:1773941560.061896:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291489152, portal 25 00010000:00000001:1.0:1773941560.061900:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000001:3.0:1773941560.061901:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941560.061906:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631466368 : -113743078085248 : ffff988d203c2d80) 00010000:00010000:1.0:1773941560.061911:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29ec lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:3.0:1773941560.061916:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c2d80 x1860112291489152/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941570 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941560.061948:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:1.0:1773941560.061952:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941560.061955:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941560.061957:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29ec lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941560.061998:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941560.062001:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941560.062006:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00000001:1.0:1773941560.062007:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000010:3.0:1773941560.062010:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac3b8. 00010000:00010000:1.0:1773941560.062013:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29ec lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:3.0:1773941560.062016:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291489152, offset 0 00000400:00000200:3.0:1773941560.062033:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00010000:1.0:1773941560.062038:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d01488a00 already unused, calling callback (ffffffffc12b4c20) 00000400:00000200:3.0:1773941560.062044:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 10000000:00000001:1.0:1773941560.062045:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941560.062052:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29ec lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941560.062053:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941560.062077:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941560.062083:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00000001:1.0:1773941560.062086:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000400:00000200:3.0:1773941560.062089:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941560.062091:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:3.0:1773941560.062094:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000020:00000001:1.0:1773941560.062097:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:3.0:1773941560.062101:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[98, 99] 00000020:00000040:1.0:1773941560.062103:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488a00 refcount=5 00000400:00000200:3.0:1773941560.062112:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000020:00000001:1.0:1773941560.062113:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182784 : -113743597368832 : ffff988d01488a00) 00000400:00000200:3.0:1773941560.062120:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [210:210:256:4294967295] 192.168.204.114@tcp LPNI seq info [100:100:8:4294967295] 00010000:00000001:1.0:1773941560.062126:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182784 : -113743597368832 : ffff988d01488a00) 00000400:00000200:3.0:1773941560.062134:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941560.062141:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941560.062146:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29ec lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941560.062153:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:3.0:1773941560.062167:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 10000000:00000001:1.0:1773941560.062178:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000800:00000010:3.0:1773941560.062179:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74800. 00000800:00000200:3.0:1773941560.062186:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:3.0:1773941560.062194:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 10000000:00010000:1.0:1773941560.062195:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29ec lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941560.062200:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74800 type 1, nob 616 niov 1 nkiov 1 10000000:01000000:1.0:1773941560.062218:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000100:00000001:3.0:1773941560.062221:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 10000000:00000040:1.0:1773941560.062222:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000400:00000200:3.0:1773941560.062227:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 10000000:00000001:1.0:1773941560.062228:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941560.062231:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294951658 00010000:00010000:1.0:1773941560.062236:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941560.062240:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000100:00000001:3.0:1773941560.062242:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294951655 00010000:00000001:1.0:1773941560.062246:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000100:00000001:3.0:1773941560.062248:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294951658 00010000:00000001:1.0:1773941560.062250:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000100:00000001:3.0:1773941560.062254:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294951655 00010000:00000001:1.0:1773941560.062256:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:3.0:1773941560.062262:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941560.062263:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941560.062270:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941560.062274:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d01488a00 with handle 0x8e86767622e29ec from hash 00000100:00000001:3.0:1773941560.062275:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941560.062289:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:1.0:1773941560.062289:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941560.062295:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941560.062297:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941560.062300:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941560.062306:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941560.062309:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000100:00000001:3.0:1773941560.062312:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000800:00000200:0.0:1773941560.062312:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00000001:1.0:1773941560.062314:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941560.062317:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00010000:00000001:1.0:1773941560.062318:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941560.062322:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941560.062328:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 10000000:00000001:1.0:1773941560.062328:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1773941560.062329:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74800. 00000100:00000001:3.0:1773941560.062334:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00010000:00010000:1.0:1773941560.062337:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29ec lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941560.062339:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941560.062339:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1773941560.062350:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941560.062356:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00010000:00000001:1.0:1773941560.062366:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:0.0:1773941560.062367:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac3b8 00010000:00000001:1.0:1773941560.062368:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941560.062372:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000400:00000010:0.0:1773941560.062372:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac3b8. 00010000:00000001:1.0:1773941560.062376:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000100:00000001:0.0:1773941560.062378:0:2393:0:(events.c:34:request_out_callback()) Process entered 00010000:00000001:1.0:1773941560.062381:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941560.062383:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941560.062387:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000200:0.0:1773941560.062387:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c2d80 x1860112291489152/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941570 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:1.0:1773941560.062390:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29ec lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:1.0:1773941560.062411:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d04dd4f00 count: 0 00000100:00000001:0.0:1773941560.062412:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941560.062419:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c2d80 x1860112291489152/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941570 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941560.062428:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941560.062430:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:1.0:1773941560.062435:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941560.062438:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941560.062442:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941560.062449:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c2680 x1860112291489024/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941576 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941560.062458:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941560.062461:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941560.062464:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d11188c00. 02000000:00000001:1.0:1773941560.062467:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941560.062469:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941560.062471:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941560.062474:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941560.062477:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d181fc200. 02000000:00000001:1.0:1773941560.062480:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941560.062482:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941560.062484:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c2680. 00000100:00000001:1.0:1773941560.062488:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941560.062490:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941560.062492:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941560.062494:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941560.062498:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941560.062502:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d04d67c80. 00000020:00000010:1.0:1773941560.062507:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d181fc200. 00000020:00000010:1.0:1773941560.062513:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d181fd800. 00000040:00000010:1.0:1773941560.062518:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d181fd600. 00000020:00000010:1.0:1773941560.062523:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941560.062529:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d181fc800. 80000000:00000010:1.0:1773941560.062533:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d054f3660. 00040000:00000010:1.0:1773941560.062540:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d181fce00. 20000000:00000010:1.0:1773941560.062545:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d181fd000. 10000000:00000040:1.0:1773941560.062550:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:1.0:1773941560.062553:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941560.062558:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d181fd800. 00000040:00000010:1.0:1773941560.062562:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d181fd600. 00000020:00000010:1.0:1773941560.062571:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941560.062575:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d181fc800. 80000000:00000010:1.0:1773941560.062579:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d054f3660. 00040000:00000010:1.0:1773941560.062584:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d181fce00. 20000000:00000010:1.0:1773941560.062588:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d181fd000. 00000020:00000010:1.0:1773941560.062591:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d181fc200. 10000000:00000010:1.0:1773941560.062594:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d04d67c80. 10000000:01000000:1.0:1773941560.062597:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941560.062601:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941560.062603:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941560.062605:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941560.062607:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941560.062609:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941560.062611:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941560.063794:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941560.063804:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941560.063808:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941560.063811:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941560.063816:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941560.063820:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941560.063827:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941560.063830:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941560.063834:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941560.063838:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324606580 00000400:00000200:0.0:1773941560.063843:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941560.063849:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xec9 [1] + 0 00000800:00000001:0.0:1773941560.063853:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941560.063860:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941560.063862:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941560.063866:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941560.063870:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941560.063874:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c2d80 x1860112291489152/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941570 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941560.063885:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c2d80 x1860112291489152/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941570 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941560.063922:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941560.063929:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941560.063934:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941560.063958:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941560.063978:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941560.063982:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941560.063983:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941560.063987:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941560.063993:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941560.063995:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941560.063998:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941560.064005:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c2d80 x1860112291489152/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941570 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941560.064014:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c2d80 x1860112291489152/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941570 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941560.064019:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941560.064020:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941560.064022:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941560.064024:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941560.064026:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941560.064028:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941560.064030:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941560.064035:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=61 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941560.064037:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941560.064039:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941560.064040:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941560.064042:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941560.064044:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941560.064046:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941560.064049:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c2d80 x1860112291489152/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941570 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941560.064066:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941560.064068:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941560.064076:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941560.064077:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941560.064083:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941560.064086:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941560.064087:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941560.064089:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941560.064092:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941560.064093:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941560.064095:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941560.064098:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941560 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941560.064100:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941560.064106:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941560.064108:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941560.064110:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941560.064112:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941560.064113:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941560.064116:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941560.064118:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941560.064120:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941560.064123:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941560.064126:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941560.064128:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941560.064447:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941560.064461:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941560.064467:0:14768:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000040:2.0:1773941560.064474:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c2d80 x1860112291489152/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941570 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1773941560.064479:0:14768:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:3.0:1773941560.064487:0:14768:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00100000:2.0:1773941560.064501:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291489152:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941560.064516:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941560.064519:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941560.064525:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c2d80 x1860112291489152/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941570 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941560.064528:0:14768:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941560.064531:0:14768:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:3.0:1773941560.064537:0:14768:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941560.064541:0:14768:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941560.064545:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941560.064549:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:3.0:1773941560.064553:0:14768:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941560.064554:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03522400. 00000001:00000001:3.0:1773941560.064557:0:14768:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941560.064561:0:14768:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941560.064562:0:14768:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941560.064563:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 10000000:00000001:3.0:1773941560.064565:0:14768:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941560.064568:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941560.064571:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941560.064575:0:14768:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000001:2.0:1773941560.064578:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941560.064582:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d1118a400. 00000100:00080000:3.0:1773941560.064584:0:14768:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:3.0:1773941560.064590:0:14768:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 02000000:00000001:2.0:1773941560.064591:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1773941560.064592:0:14768:0:(recover.c:147:ptlrpc_resend()) Process entered 02000000:00000001:2.0:1773941560.064597:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1773941560.064598:0:14768:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1773941560.064606:0:14768:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000010:2.0:1773941560.064606:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c2d80. 00000100:00000001:3.0:1773941560.064613:0:14768:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941560.064615:0:14768:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:3.0:1773941560.064618:0:14768:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00000001:2.0:1773941560.064618:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941560.064627:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941560.064632:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1773941560.064638:0:14768:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:00000001:2.0:1773941560.064638:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 10000000:00000001:3.0:1773941560.064644:0:14768:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941560.064645:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941560.064648:0:14768:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941560.064652:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:02000000:3.0:1773941560.064657:0:14768:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:2.0:1773941560.064658:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941560.064670:0:14768:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941560.064673:0:14768:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941560.064675:0:14768:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941560.064678:0:14768:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941560.064680:0:14768:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941560.068008:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d04dd4f00. 00010000:00000010:1.1:1773941560.068019:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d01488a00. 00000020:00000001:3.0:1773941560.315146:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941560.315152:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941560.315156:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941561.339109:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294951658, at 4294951680 00000100:00000010:1.0:1773941561.339125:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3060. 00000100:00000040:0.0:1773941561.339132:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294951658, at 4294951680 00000100:00000010:0.0:1773941561.339146:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f32a0. 00000020:00000001:2.0:1773941561.339165:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941561.339171:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941561.339176:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941561.467065:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941561.467084:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941561.467090:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941561.467093:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941561.467095:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941561.467097:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941561.467100:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941561.467102:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941561.467104:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:1.0:1773941561.467107:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941561.467109:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941561.467120:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c2680. 00000020:00000040:1.0:1773941561.467125:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941561.467131:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941561.467134:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941561.467140:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941561.467149:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d181fc200. 00000100:00000001:1.0:1773941561.467156:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941561.467161:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941561.467175:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941561.467183:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d04dd4f00. 00010000:00000001:1.0:1773941561.467189:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941561.467193:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d01488a00. 00000020:00000001:1.0:1773941561.467197:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941561.467199:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d01488a00 with handle 0x8e86767622e29f3 to hash 00000020:00000001:1.0:1773941561.467201:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941561.467202:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330112182784 : -113743597368832 : ffff988d01488a00) 00010000:00000001:1.0:1773941561.467205:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330112182784 : -113743597368832 : ffff988d01488a00) 00010000:00000001:1.0:1773941561.467208:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941561.467211:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941561.467214:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29f3 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941561.467223:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29f3 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941561.467232:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29f3 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941561.467236:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941561.467238:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941561.467243:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941561.467245:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330528610752 : -113743180940864 : ffff988d1a1ab9c0) 00000100:00000001:1.0:1773941561.467247:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941561.467248:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631464576 : -113743078087040 : ffff988d203c2680) 00000100:00000001:1.0:1773941561.467252:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941561.467255:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941561.467257:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941561.467258:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941561.467268:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c2680 x1860112291489280/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941561.467275:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941561.467291:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941561.467294:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941561.467296:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941561.467298:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941561.467301:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c2680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291489280:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941561.467304:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941561.467309:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941561.467311:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941561.467312:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941561.467316:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d11188c00. 02000000:00000001:1.0:1773941561.467318:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941561.467324:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941561.467328:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c5d8. 00000400:00000010:1.0:1773941561.467332:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000100:00000200:1.0:1773941561.467338:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291489280, portal 25 00000100:00000001:1.0:1773941561.467340:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941561.467341:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631464576 : -113743078087040 : ffff988d203c2680) 00000100:00000040:1.0:1773941561.467344:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c2680 x1860112291489280/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941577 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941561.467359:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941561.467362:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941561.467363:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de4c8. 00000100:00000200:1.0:1773941561.467366:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291489280, offset 0 00000400:00000200:1.0:1773941561.467370:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941561.467377:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941561.467383:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941561.467387:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941561.467390:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941561.467395:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941561.467397:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941561.467399:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[100, 99] 00000400:00000200:1.0:1773941561.467402:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:1.0:1773941561.467404:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [211:211:256:4294967295] 192.168.204.124@tcp LPNI seq info [101:101:8:4294967295] 00000400:00000200:1.0:1773941561.467422:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941561.467431:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:1.0:1773941561.467436:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:1.0:1773941561.467439:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfdf00. 00000800:00000200:1.0:1773941561.467445:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:1.0:1773941561.467451:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:1.0:1773941561.467455:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfdf00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941561.467471:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941561.467474:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941561.467476:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294953063 00000100:00000010:1.0:1773941561.467479:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3060. 00000100:00000001:1.0:1773941561.467485:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1773941561.467491:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294953063 00000100:00000010:1.0:1773941561.467494:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3660. 00000100:00000001:1.0:1773941561.467497:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1773941561.467501:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941561.467505:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941561.467507:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941561.467509:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941561.467511:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 11 seconds 00000100:00000001:1.0:1773941561.467515:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941561.467517:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941561.467609:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941561.467617:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfdf00. 00000400:00000200:0.0:1773941561.467624:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941561.467632:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941561.467639:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de4c8 00000400:00000010:0.0:1773941561.467641:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de4c8. 00000100:00000001:0.0:1773941561.467646:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941561.467653:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c2680 x1860112291489280/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941577 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941561.467665:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941561.467668:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d203c2680 x1860112291489280/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941577 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941561.467675:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941561.467677:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941561.468819:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941561.468831:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941561.468833:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941561.468835:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941561.468842:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941561.468848:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941561.468854:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941561.468858:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941561.468864:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941561.468868:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324606600 00000400:00000200:0.0:1773941561.468873:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941561.468879:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xed1 [1] + 224 00000800:00000001:0.0:1773941561.468905:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941561.468931:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941561.468935:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941561.468940:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941561.468946:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941561.468953:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c2680 x1860112291489280/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941577 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941561.468987:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d203c2680 x1860112291489280/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941577 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941561.469008:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941561.469016:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941561.469020:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941561.469098:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941561.469107:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941561.469108:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941561.469111:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:1.0:1773941561.469116:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:1.0:1773941561.469118:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:1.0:1773941561.469120:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941561.469126:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c2680 x1860112291489280/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941577 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941561.469136:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c2680 x1860112291489280/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941577 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941561.469150:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941561.469152:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941561.469155:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941561.469158:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941561.469161:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941561.469163:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941561.469166:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941561.469174:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=62 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941561.469179:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941561.469182:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941561.469187:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:1.0:1773941561.469191:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941561.469195:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941561.469199:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941561.469204:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941561.469210:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941561.469215:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941561.469218:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941561.469220:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941561.469223:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941561.469228:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d203c2680 x1860112291489280/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941577 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941561.469238:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941561.469240:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941561.469243:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941561.469247:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941561.469250:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941561.469252:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941561.469254:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941561.469256:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941561.469257:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941561.469259:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941561.469269:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941561.469272:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941561.469273:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941561.469277:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941561.469297:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 283 00000400:00000200:1.0:1773941561.469301:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941561.469305:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941561.469309:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941561.469312:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941561.469314:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941561.469316:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941561.469319:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941561.469321:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941561.469323:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941561.469326:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941561.469328:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941561.469330:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941561.469332:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941561.469333:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941561.469335:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941561.469336:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941561.469338:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941561.469341:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941561.469342:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941561.469344:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941561.469364:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941561.469366:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941561.469368:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941561.469369:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941561.469374:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c1f80. 00000020:00000040:1.0:1773941561.469377:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941561.469380:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941561.469384:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941561.469387:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941561.469390:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941561.469393:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941561.469398:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d1118a800. 00000100:00000001:1.0:1773941561.469403:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941561.469409:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c1f80 x1860112291489408/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941561.469433:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941561.469437:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941561.469441:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000708a2e6d] to pc [ptlrpcd_rcv+-1] req@ffff988d203c1f80 x1860112291489408/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941561.469465:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941561.469467:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941561.469469:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941561.469472:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941561.469474:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941561.469476:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941561.469479:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941561.469481:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941561.469483:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941561.469487:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c2680 x1860112291489280/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941577 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941561.469494:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941561.469496:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941561.469498:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941561.469500:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941561.469503:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c2680 x1860112291489280/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941577 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941561.469511:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c2680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291489280:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941561.469516:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941561.469523:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941561.469526:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941561.469528:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941561.469534:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c2680 x1860112291489280/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941577 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941561.469544:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941561.469546:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000010:1.0:1773941561.469548:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1a1ab9c0. 00000100:00000001:2.0:1773941561.469552:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941561.469555:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:2.0:1773941561.469556:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941561.469560:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:2.0:1773941561.469568:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c1f80 x1860112291489408/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941561.469570:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941561.469576:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941561.469580:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941561.469584:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488a00 refcount=4 00000020:00000001:1.0:1773941561.469587:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182784 : -113743597368832 : ffff988d01488a00) 00010000:00000001:1.0:1773941561.469596:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182784 : -113743597368832 : ffff988d01488a00) 00000100:00000001:2.0:1773941561.469606:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00010000:1.0:1773941561.469608:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29f3 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941561.469609:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941561.469617:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941561.469620:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941561.469626:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941561.469634:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291489408:192.168.204.114@tcp:250:ll_cfg_requeue.0 00010000:00000001:1.0:1773941561.469634:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941561.469643:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00010000:1.0:1773941561.469645:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29f3 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941561.469647:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941561.469651:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941561.469657:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:1.0:1773941561.469659:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 02000000:00000010:2.0:1773941561.469663:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a5c00. 00010000:00010000:1.0:1773941561.469664:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29f3 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941561.469670:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941561.469694:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941561.469695:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000010:2.0:1773941561.469699:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00010000:00000001:1.0:1773941561.469699:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941561.469701:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29f3 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941561.469705:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00010000:00000001:1.0:1773941561.469707:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000200:2.0:1773941561.469711:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291489408, portal 25 00000100:00000001:2.0:1773941561.469714:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941561.469717:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631462784 : -113743078088832 : ffff988d203c1f80) 00010000:00000001:1.0:1773941561.469719:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941561.469723:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c1f80 x1860112291489408/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941571 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941561.469724:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941561.469728:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29f3 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941561.469733:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941561.469738:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00010000:1.0:1773941561.469739:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d01488a00 already unused, calling callback (ffffffffc12b4c20) 00000400:00000010:2.0:1773941561.469741:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516198. 10000000:00000001:1.0:1773941561.469759:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000200:2.0:1773941561.469762:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291489408, offset 0 10000000:00010000:1.0:1773941561.469763:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29f3 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941561.469773:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00000001:1.0:1773941561.469776:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941561.469779:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:2.0:1773941561.469782:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000020:00000001:1.0:1773941561.469782:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941561.469786:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488a00 refcount=5 00000020:00000001:1.0:1773941561.469790:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182784 : -113743597368832 : ffff988d01488a00) 00000400:00000200:2.0:1773941561.469793:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:1.0:1773941561.469795:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182784 : -113743597368832 : ffff988d01488a00) 00000400:00000200:2.0:1773941561.469800:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:1.0:1773941561.469801:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941561.469806:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29f3 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941561.469807:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941561.469815:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 10000000:00000001:1.0:1773941561.469820:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:2.0:1773941561.469821:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 10000000:00010000:1.0:1773941561.469825:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29f3 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1773941561.469835:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:2.0:1773941561.469836:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[100, 101] 10000000:00000040:1.0:1773941561.469839:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941561.469844:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941561.469846:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00010000:00010000:1.0:1773941561.469849:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941561.469854:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:2.0:1773941561.469855:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [212:212:256:4294967295] 192.168.204.114@tcp LPNI seq info [102:102:8:4294967295] 00010000:00000001:1.0:1773941561.469858:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941561.469862:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000400:00000200:2.0:1773941561.469863:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941561.469866:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941561.469869:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941561.469872:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d01488a00 with handle 0x8e86767622e29f3 from hash 00000400:00000200:2.0:1773941561.469876:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:2.0:1773941561.469884:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00010000:00000001:1.0:1773941561.469885:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941561.469889:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000010:2.0:1773941561.469890:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0db000. 00010000:00000001:1.0:1773941561.469891:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941561.469896:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941561.469898:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941561.469902:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000800:00000200:2.0:1773941561.469903:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00010000:00000001:1.0:1773941561.469908:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941561.469910:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941561.469911:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941561.469914:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941561.469919:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00010000:00010000:1.0:1773941561.469921:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29f3 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941561.469926:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0db000 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941561.469932:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941561.469935:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941561.469936:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941561.469938:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941561.469939:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941561.469940:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941561.469941:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941561.469943:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d01488a00/0x8e86767622e29f3 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941561.469946:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941561.469953:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000040:1.0:1773941561.469953:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d04dd4f00 count: 0 00000100:00000001:2.0:1773941561.469959:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294953065 00010000:00000001:1.0:1773941561.469972:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941561.469976:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941561.469979:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941561.469986:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c2680 x1860112291489280/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941577 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941561.470003:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294953063 00000100:00000001:2.0:1773941561.470009:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294953066 00000100:00000001:2.0:1773941561.470013:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294953063 00000100:00000001:2.0:1773941561.470017:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941561.470020:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941561.470023:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941561.470026:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1773941561.470050:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941561.470056:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0db000. 00000100:00000001:2.0:1773941561.470061:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941561.470064:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000400:00000200:0.0:1773941561.470065:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1773941561.470066:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 02000000:00000001:1.0:1773941561.470086:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000400:00000200:0.0:1773941561.470095:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:2.0:1773941561.470096:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:1.0:1773941561.470096:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d11188c00. 00000100:00000001:2.0:1773941561.470105:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000200:0.0:1773941561.470105:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516198 02000000:00000001:1.0:1773941561.470107:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000400:00000010:0.0:1773941561.470113:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516198. 00000100:00000001:2.0:1773941561.470114:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941561.470115:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941561.470122:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941561.470123:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941561.470125:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:1.0:1773941561.470131:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000200:0.0:1773941561.470139:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c1f80 x1860112291489408/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941571 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1773941561.470147:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d181fc200. 00000100:00000001:0.0:1773941561.470170:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1773941561.470172:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000040:0.0:1773941561.470175:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c1f80 x1860112291489408/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941571 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1773941561.470177:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941561.470181:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c2680. 00000100:00000001:0.0:1773941561.470184:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941561.470187:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1773941561.470190:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941561.470193:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941561.470197:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941561.470201:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941561.470206:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941561.470211:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d04d67c80. 00000020:00000010:1.0:1773941561.470217:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d181fc200. 00000020:00000010:1.0:1773941561.470223:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d181fd000. 00000040:00000010:1.0:1773941561.470229:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d181fce00. 00000020:00000010:1.0:1773941561.470235:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941561.470241:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d181fc800. 80000000:00000010:1.0:1773941561.470245:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d054f3600. 00040000:00000010:1.0:1773941561.470249:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d181fd600. 20000000:00000010:1.0:1773941561.470253:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d181fd800. 10000000:00000040:1.0:1773941561.470258:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:1.0:1773941561.470260:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941561.470264:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d181fd000. 00000040:00000010:1.0:1773941561.470268:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d181fce00. 00000020:00000010:1.0:1773941561.470271:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941561.470274:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d181fc800. 80000000:00000010:1.0:1773941561.470276:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d054f3600. 00040000:00000010:1.0:1773941561.470280:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d181fd600. 20000000:00000010:1.0:1773941561.470283:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d181fd800. 00000020:00000010:1.0:1773941561.470285:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d181fc200. 10000000:00000010:1.0:1773941561.470287:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d04d67c80. 10000000:01000000:1.0:1773941561.470290:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941561.470293:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941561.470294:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941561.470296:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941561.470297:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941561.470303:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941561.470305:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941561.471986:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941561.471993:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941561.471995:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941561.471999:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941561.472019:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941561.472023:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941561.472039:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941561.472047:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941561.472050:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941561.472053:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324606680 00000400:00000200:0.0:1773941561.472056:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941561.472060:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xee1 [1] + 0 00000800:00000001:0.0:1773941561.472064:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941561.472069:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941561.472071:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941561.472073:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941561.472076:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941561.472079:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c1f80 x1860112291489408/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941571 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941561.472087:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c1f80 x1860112291489408/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941571 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941561.472100:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941561.472104:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941561.472106:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941561.472137:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941561.472140:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941561.472144:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941561.472145:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941561.472147:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941561.472156:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941561.472165:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941561.472168:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941561.472173:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c1f80 x1860112291489408/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941571 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941561.472180:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c1f80 x1860112291489408/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941571 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941561.472187:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941561.472188:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941561.472189:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941561.472191:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941561.472192:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941561.472193:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941561.472195:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941561.472198:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=62 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941561.472200:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941561.472202:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941561.472203:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941561.472205:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941561.472207:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941561.472209:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941561.472213:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c1f80 x1860112291489408/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941571 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941561.472217:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941561.472219:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941561.472221:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941561.472223:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941561.472228:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941561.472229:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941561.472230:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941561.472232:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941561.472234:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941561.472235:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941561.472237:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:2.0:1773941561.472240:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941561 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941561.472246:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941561.472252:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:2.0:1773941561.472254:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941561.472255:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941561.472257:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941561.472258:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941561.472262:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:2.0:1773941561.472264:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941561.472266:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941561.472269:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941561.472272:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:2.0:1773941561.472274:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941561.472589:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941561.472595:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941561.472603:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c1f80 x1860112291489408/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941571 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941561.472614:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291489408:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941561.472616:0:14769:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:3.0:1773941561.472623:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941561.472627:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1773941561.472630:0:14769:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00000040:3.0:1773941561.472634:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c1f80 x1860112291489408/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941571 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941561.472636:0:14769:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941561.472641:0:14769:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941561.472645:0:14769:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:2.0:1773941561.472649:0:14769:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:3.0:1773941561.472651:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:2.0:1773941561.472654:0:14769:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:3.0:1773941561.472655:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941561.472660:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a5c00. 00000001:00000001:2.0:1773941561.472661:0:14769:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:2.0:1773941561.472664:0:14769:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941561.472665:0:14769:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941561.472666:0:14769:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941561.472668:0:14769:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941561.472674:0:14769:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000001:3.0:1773941561.472693:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:2.0:1773941561.472695:0:14769:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000020:00000001:3.0:1773941561.472696:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941561.472699:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941561.472700:0:14769:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000020:00000001:3.0:1773941561.472702:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941561.472704:0:14769:0:(recover.c:147:ptlrpc_resend()) Process entered 02000000:00000010:3.0:1773941561.472706:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d1118a800. 00000100:00000001:2.0:1773941561.472707:0:14769:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941561.472710:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00080000:2.0:1773941561.472711:0:14769:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 02000000:00000001:3.0:1773941561.472713:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941561.472716:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c1f80. 00000100:00000001:2.0:1773941561.472717:0:14769:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941561.472720:0:14769:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:2.0:1773941561.472721:0:14769:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00000001:3.0:1773941561.472722:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941561.472724:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941561.472726:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941561.472729:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941561.472730:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941561.472732:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941561.472733:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1773941561.472734:0:14769:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:2.0:1773941561.472736:0:14769:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941561.472745:0:14769:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:2.0:1773941561.472750:0:14769:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:2.0:1773941561.472755:0:14769:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941561.472757:0:14769:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941561.472759:0:14769:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941561.472761:0:14769:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941561.472762:0:14769:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941561.475044:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d04dd4f00. 00010000:00000010:1.1:1773941561.475053:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d01488a00. 00000100:00000040:0.0:1773941561.595093:0:8:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (289) 00000020:00000001:3.0:1773941562.363177:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941562.363183:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941562.363187:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941562.747098:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294953066, at 4294953088 00000100:00000040:0.0:1773941562.747098:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294953065, at 4294953088 00000100:00000010:0.0:1773941562.747118:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3060. 00000100:00000010:1.0:1773941562.747121:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3660. 10000000:00000001:0.0:1773941563.067155:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1773941563.067167:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941563.067174:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941563.067177:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941563.067179:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941563.067181:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1773941563.067185:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:0.0:1773941563.067187:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:0.0:1773941563.067191:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:0.0:1773941563.067194:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1773941563.067197:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:0.0:1773941563.067212:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10835f80. 00000020:00000040:0.0:1773941563.067219:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941563.067249:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941563.067253:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941563.067260:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941563.067270:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065c5e00. 00000100:00000001:0.0:1773941563.067279:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941563.067285:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1773941563.067288:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1773941563.067297:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d21ec1400. 00010000:00000001:0.0:1773941563.067305:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1773941563.067309:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d00031600. 00000020:00000001:0.0:1773941563.067315:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:0.0:1773941563.067319:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d00031600 with handle 0x8e86767622e29fa to hash 00000020:00000001:0.0:1773941563.067322:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:0.0:1773941563.067324:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330090853888 : -113743618697728 : ffff988d00031600) 00010000:00000001:0.0:1773941563.067328:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330090853888 : -113743618697728 : ffff988d00031600) 00010000:00000001:0.0:1773941563.067333:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941563.067337:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941563.067343:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e29fa lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941563.067355:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e29fa lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941563.067370:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e29fa lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941563.067378:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1773941563.067381:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941563.067390:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb6a80. 00000100:00000001:0.0:1773941563.067392:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330139634304 : -113743569917312 : ffff988d02eb6a80) 00000100:00000001:0.0:1773941563.067396:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941563.067398:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367696768 : -113743341854848 : ffff988d10835f80) 00000100:00000001:0.0:1773941563.067404:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941563.067410:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941563.067418:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941563.067420:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941563.067441:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10835f80 x1860112291489536/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941563.067452:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941563.067455:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941563.067460:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941563.067465:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941563.067468:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941563.067472:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10835f80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291489536:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941563.067477:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941563.067485:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941563.067488:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941563.067491:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941563.067497:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d02ce7400. 02000000:00000001:0.0:1773941563.067500:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941563.067510:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941563.067515:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0014b5d8. 00000400:00000010:0.0:1773941563.067525:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833990. 00000100:00000200:0.0:1773941563.067535:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291489536, portal 25 00000100:00000001:0.0:1773941563.067538:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941563.067540:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367696768 : -113743341854848 : ffff988d10835f80) 00000100:00000040:0.0:1773941563.067546:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10835f80 x1860112291489536/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941579 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941563.067555:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941563.067558:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941563.067561:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833198. 00000100:00000200:0.0:1773941563.067565:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291489536, offset 0 00000400:00000200:0.0:1773941563.067575:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941563.067587:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941563.067598:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941563.067606:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941563.067611:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941563.067622:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941563.067626:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941563.067630:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[102, 101] 00000400:00000200:0.0:1773941563.067634:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:0.0:1773941563.067639:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [213:213:256:4294967295] 192.168.204.124@tcp LPNI seq info [103:103:8:4294967295] 00000400:00000200:0.0:1773941563.067644:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941563.067658:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:0.0:1773941563.067666:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:0.0:1773941563.067673:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a20bd00. 00000800:00000200:0.0:1773941563.067682:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:0.0:1773941563.067693:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:0.0:1773941563.067699:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a20bd00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1773941563.067726:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941563.067732:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941563.067735:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294954663 00000100:00000010:0.0:1773941563.067741:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580b40. 00000100:00000001:0.0:1773941563.067751:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1773941563.067761:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294954663 00000100:00000010:0.0:1773941563.067765:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580c60. 00000100:00000001:0.0:1773941563.067770:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1773941563.067773:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941563.067776:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941563.067780:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941563.067782:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1773941563.067786:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6a80 going to sleep for 11 seconds 00000100:00000001:0.0:1773941563.067790:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941563.067794:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941563.067850:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941563.067856:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a20bd00. 00000400:00000200:1.0:1773941563.067862:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941563.067868:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941563.067874:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833198 00000400:00000010:1.0:1773941563.067876:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833198. 00000100:00000001:1.0:1773941563.067881:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941563.067885:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10835f80 x1860112291489536/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941579 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941563.067895:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941563.067898:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d10835f80 x1860112291489536/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941579 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941563.067903:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941563.067905:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941563.070418:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941563.070431:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941563.070433:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941563.070435:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941563.070447:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941563.070451:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941563.070456:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941563.070459:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941563.070463:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941563.070467:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324606700 00000400:00000200:0.0:1773941563.070470:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941563.070474:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xee9 [1] + 224 00000800:00000001:0.0:1773941563.070481:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941563.070488:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941563.070490:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941563.070493:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941563.070497:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941563.070501:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10835f80 x1860112291489536/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941579 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941563.070510:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d10835f80 x1860112291489536/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941579 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941563.070525:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941563.070531:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941563.070533:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941563.070556:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941563.070564:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941563.070565:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941563.070568:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0014b5d8. 00000400:00000200:1.0:1773941563.070571:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833990 00000400:00000010:1.0:1773941563.070573:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833990. 00000100:00000001:1.0:1773941563.070575:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941563.070580:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10835f80 x1860112291489536/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941579 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941563.070589:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10835f80 x1860112291489536/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941579 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941563.070595:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941563.070596:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941563.070598:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941563.070600:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941563.070613:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941563.070616:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941563.070620:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941563.070628:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=64 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941563.070632:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941563.070636:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941563.070640:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:1.0:1773941563.070644:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941563.070648:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941563.070651:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941563.070656:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941563.070661:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941563.070665:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941563.070667:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941563.070669:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941563.070672:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941563.070677:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d10835f80 x1860112291489536/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941579 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941563.070686:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941563.070688:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941563.070690:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941563.070693:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941563.070696:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941563.070698:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941563.070700:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941563.070702:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941563.070703:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941563.070704:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941563.070706:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941563.070709:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941563.070711:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941563.070715:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941563.070719:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 284 00000400:00000200:1.0:1773941563.070739:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941563.070745:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941563.070750:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941563.070754:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941563.070756:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941563.070759:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941563.070763:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941563.070766:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941563.070768:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941563.070771:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941563.070774:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941563.070777:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941563.070780:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941563.070782:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941563.070784:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941563.070786:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941563.070791:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941563.070794:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941563.070798:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941563.070801:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941563.070805:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941563.070809:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941563.070812:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941563.070815:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941563.070821:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c2680. 00000020:00000040:1.0:1773941563.070824:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941563.070828:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941563.070830:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941563.070833:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941563.070836:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941563.070839:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941563.070844:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d11188000. 00000100:00000001:1.0:1773941563.070855:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941563.070863:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c2680 x1860112291489664/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941563.070873:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941563.070877:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941563.070882:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000097d1012f] to pc [ptlrpcd_rcv+-1] req@ffff988d203c2680 x1860112291489664/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941563.070904:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941563.070907:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941563.070908:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941563.070911:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941563.070913:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941563.070914:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941563.070916:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6a80 going to sleep for 1 seconds 00000100:00000001:1.0:1773941563.070918:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941563.070920:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941563.070924:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10835f80 x1860112291489536/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941579 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941563.070931:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941563.070933:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941563.070933:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941563.070938:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941563.070939:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941563.070943:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941563.070944:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941563.070951:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10835f80 x1860112291489536/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941579 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941563.070955:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c2680 x1860112291489664/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941563.070991:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10835f80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291489536:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941563.071010:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941563.071013:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941563.071015:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941563.071017:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941563.071020:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941563.071020:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1773941563.071028:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291489664:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941563.071028:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941563.071036:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1773941563.071036:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 02000000:00000001:3.0:1773941563.071041:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1773941563.071041:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1773941563.071045:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941563.071049:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10835f80 x1860112291489536/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941579 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1773941563.071050:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941563.071055:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03520000. 02000000:00000001:3.0:1773941563.071061:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941563.071069:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941563.071070:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000010:1.0:1773941563.071073:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb6a80. 00000100:00000001:1.0:1773941563.071076:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941563.071077:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:1.0:1773941563.071079:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941563.071085:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000010:3.0:1773941563.071086:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00000020:00000001:1.0:1773941563.071087:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941563.071090:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00031600 refcount=4 00000020:00000001:1.0:1773941563.071092:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090853888 : -113743618697728 : ffff988d00031600) 00000400:00000010:3.0:1773941563.071094:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00010000:00000001:1.0:1773941563.071096:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090853888 : -113743618697728 : ffff988d00031600) 00000100:00000200:3.0:1773941563.071101:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291489664, portal 25 00010000:00010000:1.0:1773941563.071102:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e29fa lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941563.071105:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941563.071108:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631464576 : -113743078087040 : ffff988d203c2680) 00010000:00000001:1.0:1773941563.071115:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000040:3.0:1773941563.071118:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c2680 x1860112291489664/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941573 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:1.0:1773941563.071120:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e29fa lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941563.071138:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:1.0:1773941563.071140:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000040:3.0:1773941563.071144:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00010000:1.0:1773941563.071145:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e29fa lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:3.0:1773941563.071147:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac110. 00000100:00000200:3.0:1773941563.071153:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291489664, offset 0 00010000:00000001:1.0:1773941563.071153:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941563.071157:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:3.0:1773941563.071160:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00010000:1.0:1773941563.071161:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e29fa lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941563.071166:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941563.071172:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:3.0:1773941563.071173:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:1.0:1773941563.071177:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941563.071178:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941563.071182:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00000001:1.0:1773941563.071182:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:3.0:1773941563.071187:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00010000:1.0:1773941563.071187:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e29fa lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941563.071192:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941563.071198:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[102, 103] 00000400:00000200:3.0:1773941563.071201:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941563.071206:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [214:214:256:4294967295] 192.168.204.114@tcp LPNI seq info [104:104:8:4294967295] 00000400:00000200:3.0:1773941563.071216:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:1.0:1773941563.071223:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d00031600 already unused, calling callback (ffffffffc12b4c20) 10000000:00000001:1.0:1773941563.071228:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:3.0:1773941563.071229:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 10000000:00010000:1.0:1773941563.071233:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e29fa lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941563.071237:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:3.0:1773941563.071245:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74800. 00010000:00000001:1.0:1773941563.071245:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941563.071249:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941563.071250:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941563.071252:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00031600 refcount=5 00000800:00000200:3.0:1773941563.071253:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000020:00000001:1.0:1773941563.071255:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090853888 : -113743618697728 : ffff988d00031600) 00010000:00000001:1.0:1773941563.071258:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090853888 : -113743618697728 : ffff988d00031600) 00010000:00000001:1.0:1773941563.071261:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000800:00000200:3.0:1773941563.071262:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00010000:00010000:1.0:1773941563.071264:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e29fa lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941563.071266:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74800 type 1, nob 616 niov 1 nkiov 1 10000000:00000001:1.0:1773941563.071275:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941563.071277:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e29fa lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1773941563.071281:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000100:00000001:3.0:1773941563.071282:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 10000000:00000040:1.0:1773941563.071284:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941563.071287:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941563.071289:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:1.0:1773941563.071291:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000100:00000001:3.0:1773941563.071292:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294954667 00010000:00000001:1.0:1773941563.071294:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941563.071297:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941563.071298:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941563.071301:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941563.071303:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941563.071306:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294954663 00000020:00000040:1.0:1773941563.071306:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d00031600 with handle 0x8e86767622e29fa from hash 00010000:00000001:1.0:1773941563.071312:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000100:00000001:3.0:1773941563.071314:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294954667 00010000:00000001:1.0:1773941563.071314:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941563.071318:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:3.0:1773941563.071320:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294954663 00000100:00000001:3.0:1773941563.071323:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941563.071327:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941563.071327:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941563.071331:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000100:00000001:3.0:1773941563.071332:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941563.071334:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000100:00000001:3.0:1773941563.071336:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:1.0:1773941563.071338:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941563.071340:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941563.071342:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941563.071343:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941563.071345:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00010000:00010000:1.0:1773941563.071346:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e29fa lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941563.071347:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1773941563.071352:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941563.071356:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00010000:00000001:1.0:1773941563.071356:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941563.071360:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:3.0:1773941563.071361:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941563.071364:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941563.071364:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941563.071367:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941563.071370:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941563.071371:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941563.071371:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941563.071373:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e29fa lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:1.0:1773941563.071379:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d21ec1400 count: 0 00010000:00000001:1.0:1773941563.071392:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941563.071394:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000800:00000200:0.0:1773941563.071394:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:1.0:1773941563.071400:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000800:00000010:0.0:1773941563.071402:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74800. 00000100:00000040:1.0:1773941563.071405:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10835f80 x1860112291489536/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941579 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1773941563.071408:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:1.0:1773941563.071415:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941563.071419:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000400:00000200:0.0:1773941563.071420:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 02000000:00000010:1.0:1773941563.071422:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d02ce7400. 00000400:00000200:0.0:1773941563.071426:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac110 02000000:00000001:1.0:1773941563.071427:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941563.071430:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941563.071433:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000400:00000010:0.0:1773941563.071433:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac110. 00000020:00000001:1.0:1773941563.071439:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941563.071442:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065c5e00. 00000100:00000001:0.0:1773941563.071442:0:2393:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:1.0:1773941563.071450:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941563.071452:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000200:0.0:1773941563.071452:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c2680 x1860112291489664/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941573 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000010:1.0:1773941563.071459:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10835f80. 00000100:00000001:1.0:1773941563.071464:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941563.071466:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941563.071470:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941563.071471:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 10000000:01000000:1.0:1773941563.071472:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941563.071477:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000100:00000040:0.0:1773941563.071481:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c2680 x1860112291489664/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941573 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000010:1.0:1773941563.071482:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d04d67c80. 00000020:00000010:1.0:1773941563.071489:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d181fc200. 00000020:00000010:1.0:1773941563.071495:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d181fd800. 00000040:00000010:1.0:1773941563.071500:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d181fd600. 00000020:00000010:1.0:1773941563.071507:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941563.071513:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d181fc800. 00000100:00000001:0.0:1773941563.071513:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941563.071519:0:2393:0:(events.c:68:request_out_callback()) Process leaving 80000000:00000010:1.0:1773941563.071521:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d054f3660. 00040000:00000010:1.0:1773941563.071527:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d181fce00. 20000000:00000010:1.0:1773941563.071531:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d181fd000. 10000000:00000040:1.0:1773941563.071535:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:1.0:1773941563.071537:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941563.071542:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d181fd800. 00000040:00000010:1.0:1773941563.071546:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d181fd600. 00000020:00000010:1.0:1773941563.071548:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941563.071551:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d181fc800. 80000000:00000010:1.0:1773941563.071553:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d054f3660. 00040000:00000010:1.0:1773941563.071557:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d181fce00. 20000000:00000010:1.0:1773941563.071559:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d181fd000. 00000020:00000010:1.0:1773941563.071560:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d181fc200. 10000000:00000010:1.0:1773941563.071562:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d04d67c80. 10000000:01000000:1.0:1773941563.071564:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941563.071567:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941563.071568:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941563.071570:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941563.071571:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941563.071572:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941563.071574:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941563.073021:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941563.073028:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941563.073030:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941563.073031:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941563.073035:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941563.073039:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941563.073043:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941563.073045:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941563.073048:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941563.073050:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324606780 00000400:00000200:0.0:1773941563.073053:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941563.073056:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xef9 [1] + 0 00000800:00000001:0.0:1773941563.073061:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941563.073068:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941563.073071:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941563.073075:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941563.073080:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941563.073085:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c2680 x1860112291489664/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941573 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941563.073095:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c2680 x1860112291489664/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941573 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941563.073112:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941563.073117:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941563.073121:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941563.073140:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941563.073143:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941563.073147:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941563.073148:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941563.073150:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941563.073154:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941563.073155:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941563.073158:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941563.073163:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c2680 x1860112291489664/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941573 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941563.073171:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c2680 x1860112291489664/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941573 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941563.073177:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941563.073178:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941563.073180:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941563.073181:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941563.073182:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941563.073184:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941563.073185:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941563.073188:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=64 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941563.073190:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941563.073192:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941563.073200:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941563.073209:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941563.073211:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941563.073213:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941563.073217:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c2680 x1860112291489664/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941573 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941563.073223:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941563.073224:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941563.073226:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941563.073228:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941563.073232:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941563.073234:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941563.073235:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941563.073236:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941563.073238:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941563.073240:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941563.073242:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941563.073245:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941563 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941563.073247:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941563.073253:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941563.073254:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941563.073256:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941563.073257:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941563.073259:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941563.073261:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941563.073263:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941563.073265:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941563.073268:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941563.073271:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941563.073273:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941563.073533:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941563.073537:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941563.073544:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c2680 x1860112291489664/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941573 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941563.073548:0:14770:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:2.0:1773941563.073554:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291489664:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00080000:3.0:1773941563.073559:0:14770:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00000001:2.0:1773941563.073559:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941563.073563:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00100000:3.0:1773941563.073565:0:14770:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000040:2.0:1773941563.073568:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c2680 x1860112291489664/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941573 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941563.073575:0:14770:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941563.073578:0:14770:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000001:2.0:1773941563.073578:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941563.073580:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:3.0:1773941563.073583:0:14770:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000010:2.0:1773941563.073584:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03520000. 00000001:00000001:3.0:1773941563.073589:0:14770:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941563.073593:0:14770:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941563.073597:0:14770:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:2.0:1773941563.073597:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:3.0:1773941563.073599:0:14770:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000001:2.0:1773941563.073600:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:3.0:1773941563.073602:0:14770:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941563.073603:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 10000000:00000001:3.0:1773941563.073606:0:14770:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941563.073607:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941563.073608:0:14770:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000010:2.0:1773941563.073610:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d11188000. 00000100:00080000:3.0:1773941563.073614:0:14770:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 02000000:00000001:2.0:1773941563.073614:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941563.073615:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1773941563.073619:0:14770:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000010:2.0:1773941563.073619:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c2680. 00000100:00000001:3.0:1773941563.073624:0:14770:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:2.0:1773941563.073626:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941563.073627:0:14770:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941563.073629:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1773941563.073631:0:14770:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941563.073632:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941563.073637:0:14770:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941563.073639:0:14770:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000100:00000001:2.0:1773941563.073640:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 10000000:00000040:3.0:1773941563.073641:0:14770:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00000001:2.0:1773941563.073644:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941563.073646:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1773941563.073655:0:14770:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:00000001:2.0:1773941563.073655:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941563.073658:0:14770:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941563.073659:0:14770:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:3.0:1773941563.073662:0:14770:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:3.0:1773941563.073665:0:14770:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941563.073667:0:14770:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941563.073669:0:14770:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941563.073671:0:14770:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941563.073672:0:14770:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941563.076989:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d21ec1400. 00010000:00000010:1.1:1773941563.077010:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d00031600. 00000020:00000001:2.0:1773941563.387152:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941563.387161:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941563.387165:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941564.283186:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941564.283338:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941564.283344:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941564.283346:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941564.283348:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941564.283349:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941564.283352:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941564.283353:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941564.283356:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:1.0:1773941564.283358:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941564.283359:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941564.283370:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c2300. 00000020:00000040:1.0:1773941564.283374:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941564.283378:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941564.283381:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941564.283386:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941564.283393:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d181fc200. 00000100:00000001:1.0:1773941564.283398:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941564.283402:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941564.283404:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941564.283412:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d04dd4e00. 00010000:00000001:1.0:1773941564.283417:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941564.283419:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d01488e00. 00000020:00000001:1.0:1773941564.283423:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941564.283425:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d01488e00 with handle 0x8e86767622e2a01 to hash 00000020:00000001:1.0:1773941564.283427:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941564.283428:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330112183808 : -113743597367808 : ffff988d01488e00) 00010000:00000001:1.0:1773941564.283431:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330112183808 : -113743597367808 : ffff988d01488e00) 00010000:00000001:1.0:1773941564.283434:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941564.283436:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941564.283440:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a01 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941564.283449:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a01 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941564.283475:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a01 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941564.283479:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941564.283481:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941564.283487:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941564.283489:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330528610752 : -113743180940864 : ffff988d1a1ab9c0) 00000100:00000001:1.0:1773941564.283491:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941564.283492:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631463680 : -113743078087936 : ffff988d203c2300) 00000100:00000001:1.0:1773941564.283496:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941564.283499:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941564.283501:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941564.283503:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941564.283511:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c2300 x1860112291489792/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941564.283518:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941564.283519:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941564.283522:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941564.283524:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941564.283526:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941564.283528:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c2300 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291489792:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941564.283531:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941564.283535:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941564.283537:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941564.283539:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941564.283543:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d11188800. 02000000:00000001:1.0:1773941564.283544:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941564.283550:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941564.283554:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c5d8. 00000400:00000010:1.0:1773941564.283559:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000100:00000200:1.0:1773941564.283564:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291489792, portal 25 00000100:00000001:1.0:1773941564.283566:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941564.283571:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631463680 : -113743078087936 : ffff988d203c2300) 00000100:00000040:1.0:1773941564.283576:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c2300 x1860112291489792/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941580 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941564.283581:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941564.283583:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941564.283585:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de550. 00000100:00000200:1.0:1773941564.283587:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291489792, offset 0 00000400:00000200:1.0:1773941564.283593:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941564.283601:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941564.283607:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941564.283611:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941564.283614:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941564.283617:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941564.283619:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941564.283622:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[104, 103] 00000400:00000200:1.0:1773941564.283625:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:1.0:1773941564.283628:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [215:215:256:4294967295] 192.168.204.124@tcp LPNI seq info [105:105:8:4294967295] 00000400:00000200:1.0:1773941564.283631:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941564.283638:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:1.0:1773941564.283642:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:1.0:1773941564.283646:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfd800. 00000800:00000200:1.0:1773941564.283651:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:1.0:1773941564.283657:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:1.0:1773941564.283661:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfd800 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941564.283683:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941564.283686:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941564.283688:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294955879 00000100:00000001:1.0:1773941564.283696:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294954667 00000100:00000001:1.0:1773941564.283700:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294955879 00000100:00000001:1.0:1773941564.283703:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294954667 00000100:00000001:1.0:1773941564.283709:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941564.283711:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941564.283713:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941564.283715:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941564.283717:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 11 seconds 00000100:00000001:1.0:1773941564.283719:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941564.283721:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941564.283902:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941564.283911:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfd800. 00000400:00000200:0.0:1773941564.283919:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941564.283929:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941564.283937:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de550 00000400:00000010:0.0:1773941564.283950:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de550. 00000100:00000001:0.0:1773941564.283955:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941564.284035:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c2300 x1860112291489792/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941580 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941564.284049:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941564.284053:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d203c2300 x1860112291489792/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941580 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941564.284061:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941564.284063:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941564.286241:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941564.286255:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941564.286259:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941564.286261:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941564.286269:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941564.286275:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941564.286283:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941564.286288:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941564.286294:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941564.286299:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324606800 00000400:00000200:0.0:1773941564.286305:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941564.286315:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xf01 [1] + 224 00000800:00000001:0.0:1773941564.286327:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941564.286339:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941564.286343:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941564.286348:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941564.286355:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941564.286362:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c2300 x1860112291489792/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941580 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941564.286380:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d203c2300 x1860112291489792/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941580 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941564.286414:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941564.286436:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941564.286440:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941564.286469:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941564.286479:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941564.286482:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941564.286485:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:1.0:1773941564.286490:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:1.0:1773941564.286492:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:1.0:1773941564.286495:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941564.286502:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c2300 x1860112291489792/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941580 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941564.286514:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c2300 x1860112291489792/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941580 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941564.286522:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941564.286523:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941564.286526:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941564.286530:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941564.286533:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941564.286535:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941564.286538:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941564.286546:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=65 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941564.286550:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941564.286553:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941564.286558:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:1.0:1773941564.286561:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941564.286564:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941564.286567:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941564.286572:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941564.286577:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941564.286589:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941564.286592:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941564.286594:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941564.286597:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941564.286602:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d203c2300 x1860112291489792/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941580 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941564.286614:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941564.286616:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941564.286619:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941564.286623:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941564.286627:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941564.286630:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941564.286633:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941564.286635:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941564.286636:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941564.286639:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941564.286641:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941564.286645:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941564.286648:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941564.286652:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941564.286657:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 286 00000400:00000200:1.0:1773941564.286664:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941564.286672:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941564.286678:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941564.286683:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941564.286687:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941564.286691:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941564.286696:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941564.286699:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941564.286703:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941564.286706:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941564.286708:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941564.286711:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941564.286721:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941564.286724:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941564.286726:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941564.286728:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941564.286731:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941564.286734:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941564.286737:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941564.286740:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941564.286743:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941564.286746:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941564.286748:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941564.286750:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941564.286755:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c1880. 00000020:00000040:1.0:1773941564.286758:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941564.286762:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941564.286764:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941564.286767:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941564.286783:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941564.286787:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941564.286794:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d11189000. 00000100:00000001:1.0:1773941564.286800:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941564.286806:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c1880 x1860112291489920/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941564.286815:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941564.286824:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941564.286829:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008b2304da] to pc [ptlrpcd_rcv+-1] req@ffff988d203c1880 x1860112291489920/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941564.286850:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941564.286852:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941564.286854:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941564.286858:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941564.286860:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941564.286867:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941564.286870:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941564.286872:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941564.286874:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941564.286879:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c2300 x1860112291489792/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941580 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941564.286886:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941564.286889:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941564.286891:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941564.286894:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941564.286898:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c2300 x1860112291489792/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941580 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941564.286906:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c2300 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291489792:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941564.286910:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941564.286913:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941564.286916:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941564.286917:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941564.286921:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c2300 x1860112291489792/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941580 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941564.286927:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941564.286930:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941564.286933:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941564.286935:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941564.286939:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:1.0:1773941564.286939:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000001:2.0:1773941564.286948:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00010000:00000001:1.0:1773941564.286948:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941564.286953:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1773941564.286954:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000020:00000040:1.0:1773941564.286958:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488e00 refcount=4 00000100:00000040:2.0:1773941564.287009:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c1880 x1860112291489920/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1773941564.287019:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112183808 : -113743597367808 : ffff988d01488e00) 00010000:00000001:1.0:1773941564.287028:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112183808 : -113743597367808 : ffff988d01488e00) 00000100:00000001:2.0:1773941564.287029:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941564.287035:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941564.287040:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00010000:00010000:1.0:1773941564.287040:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a01 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941564.287043:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941564.287046:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941564.287052:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291489920:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941564.287062:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941564.287067:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00000001:1.0:1773941564.287070:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 02000000:00000001:2.0:1773941564.287071:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941564.287077:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00010000:1.0:1773941564.287081:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a01 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000010:2.0:1773941564.287085:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a4400. 02000000:00000001:2.0:1773941564.287091:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941564.287095:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000400:00000200:2.0:1773941564.287099:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:1.0:1773941564.287101:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a01 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941564.287104:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00010000:00000001:1.0:1773941564.287113:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000010:2.0:1773941564.287114:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00010000:00000001:1.0:1773941564.287117:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941564.287121:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a01 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:2.0:1773941564.287122:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291489920, portal 25 00000100:00000001:2.0:1773941564.287129:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941564.287132:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631460992 : -113743078090624 : ffff988d203c1880) 00010000:00000001:1.0:1773941564.287135:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941564.287140:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941564.287141:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c1880 x1860112291489920/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941574 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941564.287146:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941564.287151:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a01 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941564.287156:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941564.287164:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00010000:1.0:1773941564.287165:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d01488e00 already unused, calling callback (ffffffffc12b4c20) 00000400:00000010:2.0:1773941564.287168:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516088. 10000000:00000001:1.0:1773941564.287170:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000200:2.0:1773941564.287174:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291489920, offset 0 10000000:00010000:1.0:1773941564.287176:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a01 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941564.287187:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00000001:1.0:1773941564.287190:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941564.287192:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941564.287193:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941564.287194:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488e00 refcount=5 00000020:00000001:1.0:1773941564.287196:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112183808 : -113743597367808 : ffff988d01488e00) 00010000:00000001:1.0:1773941564.287201:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112183808 : -113743597367808 : ffff988d01488e00) 00010000:00000001:1.0:1773941564.287206:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:2.0:1773941564.287211:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:1.0:1773941564.287220:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a01 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941564.287226:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 10000000:00000001:1.0:1773941564.287230:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941564.287234:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a01 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941564.287239:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 10000000:01000000:1.0:1773941564.287246:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:2.0:1773941564.287252:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 10000000:00000040:1.0:1773941564.287253:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941564.287257:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941564.287261:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000400:00000200:2.0:1773941564.287263:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941564.287264:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941564.287270:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000400:00000200:2.0:1773941564.287272:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941564.287276:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000400:00000200:2.0:1773941564.287283:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[104, 105] 00010000:00000001:1.0:1773941564.287284:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941564.287288:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941564.287295:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000020:00000040:1.0:1773941564.287297:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d01488e00 with handle 0x8e86767622e2a01 from hash 00000400:00000200:2.0:1773941564.287306:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [216:216:256:4294967295] 192.168.204.114@tcp LPNI seq info [106:106:8:4294967295] 00010000:00000001:1.0:1773941564.287306:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941564.287311:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941564.287316:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:2.0:1773941564.287321:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941564.287321:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941564.287326:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941564.287332:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941564.287340:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941564.287344:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:2.0:1773941564.287346:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00010000:00000001:1.0:1773941564.287348:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941564.287356:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941564.287363:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00010000:00010000:1.0:1773941564.287364:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a01 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000010:2.0:1773941564.287373:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0db900. 00010000:00000001:1.0:1773941564.287375:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941564.287380:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941564.287382:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941564.287384:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941564.287388:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941564.287389:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941564.287397:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941564.287399:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a01 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941564.287401:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00010000:00000040:1.0:1773941564.287409:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d04dd4e00 count: 0 00000800:00000200:2.0:1773941564.287426:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00010000:00000001:1.0:1773941564.287428:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941564.287433:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000800:00000200:2.0:1773941564.287434:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0db900 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:1.0:1773941564.287436:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941564.287445:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c2300 x1860112291489792/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941580 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941564.287454:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941564.287458:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1773941564.287459:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941564.287462:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d11188800. 02000000:00000001:1.0:1773941564.287469:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000400:00000200:2.0:1773941564.287473:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000020:00000001:1.0:1773941564.287474:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941564.287480:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941564.287482:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294955883 00000020:00000001:1.0:1773941564.287484:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941564.287490:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d181fc200. 02000000:00000001:1.0:1773941564.287496:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1773941564.287498:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294955879 02000000:00000001:1.0:1773941564.287499:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941564.287502:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c2300. 00000100:00000001:1.0:1773941564.287511:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941564.287513:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294955883 00000100:00000001:1.0:1773941564.287513:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941564.287517:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941564.287522:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294955879 10000000:01000000:1.0:1773941564.287523:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941564.287530:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000100:00000001:2.0:1773941564.287531:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941564.287536:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 10000000:00000010:1.0:1773941564.287537:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d04d679c0. 00000100:00000001:2.0:1773941564.287540:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941564.287542:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1773941564.287546:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1773941564.287547:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d181fc200. 00000800:00000010:0.0:1773941564.287548:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0db900. 00000020:00000010:1.0:1773941564.287558:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d181fd000. 00000400:00000200:0.0:1773941564.287558:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1773941564.287559:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941564.287564:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000040:00000010:1.0:1773941564.287568:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d181fce00. 00000100:00000001:2.0:1773941564.287569:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941564.287573:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000200:0.0:1773941564.287575:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:2.0:1773941564.287577:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941564.287579:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d18b18000. 00000400:00000200:0.0:1773941564.287581:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516088 00000400:00000010:0.0:1773941564.287586:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516088. 00000020:00000010:1.0:1773941564.287588:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d181fc800. 00000100:00000001:2.0:1773941564.287595:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941564.287595:0:2393:0:(events.c:34:request_out_callback()) Process entered 80000000:00000010:1.0:1773941564.287598:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d054f3660. 00000100:00000200:0.0:1773941564.287606:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c1880 x1860112291489920/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941574 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00040000:00000010:1.0:1773941564.287608:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d181fd600. 20000000:00000010:1.0:1773941564.287622:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d181fd800. 00000100:00000001:0.0:1773941564.287623:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941564.287627:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c1880 x1860112291489920/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941574 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941564.287639:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941564.287641:0:2393:0:(events.c:68:request_out_callback()) Process leaving 10000000:00000040:1.0:1773941564.287646:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:1.0:1773941564.287653:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941564.287663:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d181fd000. 00000040:00000010:1.0:1773941564.287671:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d181fce00. 00000020:00000010:1.0:1773941564.287678:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941564.287685:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d181fc800. 80000000:00000010:1.0:1773941564.287691:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d054f3660. 00040000:00000010:1.0:1773941564.287698:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d181fd600. 20000000:00000010:1.0:1773941564.287703:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d181fd800. 00000020:00000010:1.0:1773941564.287707:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d181fc200. 10000000:00000010:1.0:1773941564.287711:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d04d679c0. 10000000:01000000:1.0:1773941564.287715:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941564.287720:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941564.287724:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941564.287727:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941564.287729:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941564.287731:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941564.287735:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941564.289505:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941564.289516:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941564.289519:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941564.289523:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941564.289530:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941564.289539:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941564.289548:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941564.289554:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941564.289559:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941564.289564:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324606880 00000400:00000200:0.0:1773941564.289570:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941564.289576:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xf11 [1] + 0 00000800:00000001:0.0:1773941564.289583:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941564.289592:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941564.289596:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941564.289601:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941564.289607:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941564.289623:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c1880 x1860112291489920/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941574 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941564.289639:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c1880 x1860112291489920/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941574 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941564.289664:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941564.289674:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941564.289679:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941564.289719:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941564.289724:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941564.289729:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941564.289731:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941564.289735:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941564.289740:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941564.289744:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941564.289747:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941564.289755:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c1880 x1860112291489920/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941574 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941564.289769:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c1880 x1860112291489920/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941574 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941564.289779:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941564.289782:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941564.289785:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941564.289787:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941564.289789:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941564.289792:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941564.289795:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941564.289800:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=65 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941564.289805:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941564.289808:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941564.289811:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941564.289814:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941564.289818:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941564.289821:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941564.289827:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c1880 x1860112291489920/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941574 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941564.289837:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941564.289841:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941564.289860:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941564.289864:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941564.289873:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941564.289878:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941564.289893:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941564.289897:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941564.289901:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941564.289905:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941564.289910:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:2.0:1773941564.289919:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941564 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941564.289925:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941564.289936:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:2.0:1773941564.289940:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941564.289944:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941564.289948:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941564.289951:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941564.289959:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:2.0:1773941564.289963:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941564.289968:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941564.290056:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941564.290062:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:2.0:1773941564.290067:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941564.290325:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941564.290331:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941564.290339:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c1880 x1860112291489920/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941574 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941564.290354:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291489920:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941564.290362:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941564.290365:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941564.290366:0:14772:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000040:2.0:1773941564.290370:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c1880 x1860112291489920/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941574 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941564.290380:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941564.290383:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00080000:0.0:1773941564.290384:0:14772:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 02000000:00000010:2.0:1773941564.290387:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a4400. 02000000:00000001:2.0:1773941564.290393:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00100000:0.0:1773941564.290393:0:14772:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:0.0:1773941564.290400:0:14772:0:(obd_class.h:1215:obd_import_event()) Process entered 00000020:00000001:2.0:1773941564.290405:0:2400:0:(genops.c:1236:class_import_put()) Process entered 10000000:01000000:0.0:1773941564.290407:0:14772:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000020:00000040:2.0:1773941564.290409:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941564.290416:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:0.0:1773941564.290416:0:14772:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000010:2.0:1773941564.290421:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d11189000. 00000001:00000001:0.0:1773941564.290424:0:14772:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941564.290426:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941564.290431:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941564.290433:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c1880. 00000001:00000001:0.0:1773941564.290434:0:14772:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941564.290438:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000001:00000001:0.0:1773941564.290440:0:14772:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:2.0:1773941564.290442:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:0.0:1773941564.290444:0:14772:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941564.290446:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:0.0:1773941564.290450:0:14772:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941564.290453:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 10000000:00000001:0.0:1773941564.290455:0:14772:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941564.290457:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941564.290461:0:14772:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941564.290462:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941564.290467:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941564.290471:0:14772:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:0.0:1773941564.290476:0:14772:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:0.0:1773941564.290480:0:14772:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:0.0:1773941564.290484:0:14772:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941564.290488:0:14772:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:0.0:1773941564.290491:0:14772:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941564.290493:0:14772:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941564.290495:0:14772:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:0.0:1773941564.290522:0:14772:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941564.290525:0:14772:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941564.290528:0:14772:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941564.290534:0:14772:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:0.0:1773941564.290541:0:14772:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941564.290545:0:14772:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941564.290557:0:14772:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:0.0:1773941564.290561:0:14772:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941564.290564:0:14772:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941564.293020:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d04dd4e00. 00010000:00000010:1.1:1773941564.293029:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d01488e00. 00000020:00000001:2.0:1773941564.411181:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941564.411190:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941564.411196:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1773941565.435160:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941565.435168:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941565.435173:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941565.563089:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294955883, at 4294955904 00000100:00000040:0.0:1773941565.563089:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294955883, at 4294955904 00000100:00000010:1.0:1773941565.563110:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580c60. 00000100:00000010:0.0:1773941565.563114:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580b40. 10000000:00000001:1.0:1773941565.883134:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941565.883146:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941565.883152:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941565.883154:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941565.883156:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941565.883158:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941565.883161:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941565.883163:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941565.883174:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:1.0:1773941565.883176:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941565.883178:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941565.883194:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c2300. 00000020:00000040:1.0:1773941565.883199:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941565.883205:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941565.883230:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941565.883239:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941565.883250:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d181fc200. 00000100:00000001:1.0:1773941565.883260:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941565.883267:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941565.883270:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941565.883282:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d04dd4e00. 00010000:00000001:1.0:1773941565.883290:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941565.883295:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d01488e00. 00000020:00000001:1.0:1773941565.883300:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941565.883304:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d01488e00 with handle 0x8e86767622e2a08 to hash 00000020:00000001:1.0:1773941565.883307:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941565.883309:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330112183808 : -113743597367808 : ffff988d01488e00) 00010000:00000001:1.0:1773941565.883313:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330112183808 : -113743597367808 : ffff988d01488e00) 00010000:00000001:1.0:1773941565.883319:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941565.883323:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941565.883328:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a08 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941565.883338:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a08 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941565.883350:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a08 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941565.883355:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941565.883357:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941565.883366:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941565.883367:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330528610752 : -113743180940864 : ffff988d1a1ab9c0) 00000100:00000001:1.0:1773941565.883370:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941565.883371:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631463680 : -113743078087936 : ffff988d203c2300) 00000100:00000001:1.0:1773941565.883377:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941565.883381:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941565.883383:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941565.883390:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941565.883403:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c2300 x1860112291490048/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941565.883411:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941565.883413:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941565.883417:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941565.883420:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941565.883422:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941565.883424:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c2300 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291490048:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941565.883429:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941565.883436:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941565.883438:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941565.883441:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941565.883445:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d11188800. 02000000:00000001:1.0:1773941565.883447:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941565.883456:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941565.883460:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c5d8. 00000400:00000010:1.0:1773941565.883468:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000100:00000200:1.0:1773941565.883476:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291490048, portal 25 00000100:00000001:1.0:1773941565.883478:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941565.883479:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631463680 : -113743078087936 : ffff988d203c2300) 00000100:00000040:1.0:1773941565.883483:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c2300 x1860112291490048/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941581 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941565.883489:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941565.883492:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941565.883494:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de7f8. 00000100:00000200:1.0:1773941565.883496:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291490048, offset 0 00000400:00000200:1.0:1773941565.883504:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941565.883513:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941565.883521:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941565.883528:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941565.883532:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941565.883536:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941565.883542:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941565.883545:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[106, 105] 00000400:00000200:1.0:1773941565.883548:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:1.0:1773941565.883552:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [217:217:256:4294967295] 192.168.204.124@tcp LPNI seq info [107:107:8:4294967295] 00000400:00000200:1.0:1773941565.883556:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941565.883566:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:1.0:1773941565.883571:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:1.0:1773941565.883576:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfda00. 00000800:00000200:1.0:1773941565.883584:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:1.0:1773941565.883593:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:1.0:1773941565.883598:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfda00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941565.883621:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941565.883625:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941565.883628:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294957479 00000100:00000010:1.0:1773941565.883631:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3660. 00000100:00000001:1.0:1773941565.883639:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1773941565.883647:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294957479 00000100:00000010:1.0:1773941565.883650:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3600. 00000100:00000001:1.0:1773941565.883653:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1773941565.883657:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941565.883659:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941565.883662:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941565.883664:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941565.883667:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 11 seconds 00000100:00000001:1.0:1773941565.883670:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941565.883673:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941565.883756:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941565.883762:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfda00. 00000400:00000200:0.0:1773941565.883768:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941565.883774:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941565.883779:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de7f8 00000400:00000010:0.0:1773941565.883781:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de7f8. 00000100:00000001:0.0:1773941565.883785:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941565.883790:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c2300 x1860112291490048/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941581 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941565.883800:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941565.883803:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d203c2300 x1860112291490048/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941581 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941565.883809:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941565.883811:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941565.887345:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941565.887380:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941565.887384:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941565.887387:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941565.887392:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941565.887397:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941565.887402:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941565.887406:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941565.887410:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941565.887414:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324606900 00000400:00000200:0.0:1773941565.887417:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941565.887421:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xf19 [1] + 224 00000800:00000001:0.0:1773941565.887428:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941565.887436:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941565.887445:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941565.887448:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941565.887452:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941565.887456:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c2300 x1860112291490048/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941581 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941565.887465:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d203c2300 x1860112291490048/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941581 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941565.887479:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941565.887485:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941565.887488:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941565.887516:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941565.887524:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941565.887525:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941565.887528:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:1.0:1773941565.887531:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:1.0:1773941565.887533:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:1.0:1773941565.887543:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941565.887548:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c2300 x1860112291490048/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941581 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941565.887557:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c2300 x1860112291490048/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941581 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941565.887563:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941565.887564:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941565.887567:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941565.887569:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941565.887571:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941565.887573:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941565.887575:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941565.887582:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=66 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941565.887585:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941565.887588:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941565.887592:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:1.0:1773941565.887595:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941565.887598:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941565.887600:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941565.887605:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941565.887609:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941565.887612:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941565.887614:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941565.887615:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941565.887617:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941565.887620:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d203c2300 x1860112291490048/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941581 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941565.887628:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941565.887629:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941565.887631:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941565.887633:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941565.887636:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941565.887642:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941565.887645:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941565.887647:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941565.887648:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941565.887649:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941565.887650:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941565.887653:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941565.887655:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941565.887657:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941565.887659:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 287 00000400:00000200:1.0:1773941565.887663:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941565.887666:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941565.887670:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941565.887673:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941565.887675:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941565.887676:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941565.887679:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941565.887681:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941565.887683:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941565.887686:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941565.887687:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941565.887689:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941565.887691:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941565.887692:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941565.887693:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941565.887694:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941565.887697:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941565.887699:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941565.887701:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941565.887703:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941565.887706:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941565.887708:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941565.887714:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941565.887716:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941565.887719:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c3b80. 00000020:00000040:1.0:1773941565.887721:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941565.887724:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941565.887726:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941565.887728:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941565.887730:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941565.887732:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941565.887736:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d1118a000. 00000100:00000001:1.0:1773941565.887739:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941565.887744:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c3b80 x1860112291490176/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941565.887750:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941565.887752:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941565.887756:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e15b8eeb] to pc [ptlrpcd_rcv+-1] req@ffff988d203c3b80 x1860112291490176/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941565.887775:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941565.887778:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941565.887779:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941565.887782:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941565.887784:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941565.887785:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941565.887787:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941565.887788:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941565.887790:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941565.887793:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c2300 x1860112291490048/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941581 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941565.887799:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941565.887802:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941565.887805:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941565.887806:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941565.887810:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c2300 x1860112291490048/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941581 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941565.887815:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941565.887820:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941565.887823:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00100000:1.0:1773941565.887824:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c2300 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291490048:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941565.887830:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941565.887831:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c3b80 x1860112291490176/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941565.887833:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941565.887839:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941565.887842:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941565.887846:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941565.887848:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941565.887848:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c2300 x1860112291490048/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941581 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1773941565.887854:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941565.887859:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941565.887859:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941565.887864:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1a1ab9c0. 02000000:00000001:2.0:1773941565.887866:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941565.887868:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941565.887871:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00100000:2.0:1773941565.887874:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291490176:192.168.204.114@tcp:250:ll_cfg_requeue.0 00010000:00000001:1.0:1773941565.887875:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941565.887879:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000001:2.0:1773941565.887883:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000001:1.0:1773941565.887883:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941565.887888:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488e00 refcount=4 02000000:00000001:2.0:1773941565.887889:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:1.0:1773941565.887890:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112183808 : -113743597367808 : ffff988d01488e00) 02000000:00000001:2.0:1773941565.887894:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941565.887895:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112183808 : -113743597367808 : ffff988d01488e00) 02000000:00000001:2.0:1773941565.887901:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00010000:1.0:1773941565.887902:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a08 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000010:2.0:1773941565.887910:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a4400. 00010000:00000001:1.0:1773941565.887910:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00010000:00010000:1.0:1773941565.887915:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a08 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941565.887920:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941565.887923:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a08 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941565.887925:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941565.887932:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:2.0:1773941565.887935:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941565.887940:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00010000:00000001:1.0:1773941565.887941:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941565.887985:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a08 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941565.887991:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941565.887993:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941565.887996:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941565.887998:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a08 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941565.888003:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d01488e00 already unused, calling callback (ffffffffc12b4c20) 10000000:00000001:1.0:1773941565.888006:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941565.888008:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a08 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941565.888009:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000200:2.0:1773941565.888015:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291490176, portal 25 00010000:00000001:1.0:1773941565.888017:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000100:00000001:2.0:1773941565.888019:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00000001:1.0:1773941565.888020:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000001:2.0:1773941565.888023:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631469952 : -113743078081664 : ffff988d203c3b80) 00000020:00000001:1.0:1773941565.888023:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941565.888026:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488e00 refcount=5 00000020:00000001:1.0:1773941565.888029:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112183808 : -113743597367808 : ffff988d01488e00) 00010000:00000001:1.0:1773941565.888032:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112183808 : -113743597367808 : ffff988d01488e00) 00000100:00000040:2.0:1773941565.888033:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c3b80 x1860112291490176/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941575 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941565.888036:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941565.888040:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a08 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941565.888045:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941565.888050:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 10000000:00000001:1.0:1773941565.888051:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941565.888054:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a08 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941565.888055:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045165d8. 00000100:00000200:2.0:1773941565.888061:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291490176, offset 0 10000000:01000000:1.0:1773941565.888064:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941565.888067:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000400:00000200:2.0:1773941565.888069:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 10000000:00000001:1.0:1773941565.888070:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941565.888074:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000400:00000200:2.0:1773941565.888077:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941565.888078:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:2.0:1773941565.888082:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941565.888087:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:1.0:1773941565.888088:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000400:00000200:2.0:1773941565.888091:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00000001:1.0:1773941565.888091:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000400:00000200:2.0:1773941565.888095:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941565.888095:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941565.888099:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941565.888100:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000020:00000040:1.0:1773941565.888104:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d01488e00 with handle 0x8e86767622e2a08 from hash 00000400:00000200:2.0:1773941565.888105:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[106, 107] 00010000:00000001:1.0:1773941565.888109:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000400:00000200:2.0:1773941565.888111:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00010000:00000001:1.0:1773941565.888113:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941565.888116:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:2.0:1773941565.888117:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [218:218:256:4294967295] 192.168.204.114@tcp LPNI seq info [108:108:8:4294967295] 00010000:00000001:1.0:1773941565.888118:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941565.888123:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000400:00000200:2.0:1773941565.888124:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941565.888126:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941565.888129:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941565.888132:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941565.888134:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941565.888135:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 10000000:00000001:1.0:1773941565.888137:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941565.888142:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00010000:00010000:1.0:1773941565.888142:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a08 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000010:2.0:1773941565.888149:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0db600. 00010000:00000001:1.0:1773941565.888150:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941565.888152:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941565.888155:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941565.888157:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000800:00000200:2.0:1773941565.888158:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00010000:00000001:1.0:1773941565.888159:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941565.888163:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941565.888164:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941565.888166:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a08 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941565.888172:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00010000:00000040:1.0:1773941565.888174:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d04dd4e00 count: 0 00000800:00000200:2.0:1773941565.888180:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0db600 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941565.888188:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941565.888191:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941565.888193:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941565.888196:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941565.888197:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c2300 x1860112291490048/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941581 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941565.888205:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941565.888209:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000400:00000200:2.0:1773941565.888211:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941565.888215:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294957484 02000000:00000010:1.0:1773941565.888220:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d11188800. 02000000:00000001:1.0:1773941565.888224:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1773941565.888226:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294957479 00000020:00000001:1.0:1773941565.888227:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941565.888230:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941565.888234:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941565.888235:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294957484 02000000:00000010:1.0:1773941565.888237:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d181fc200. 00000100:00000001:2.0:1773941565.888243:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294957479 02000000:00000001:1.0:1773941565.888245:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1773941565.888248:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941565.888250:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1773941565.888251:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941565.888255:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941565.888255:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c2300. 00000100:00000001:2.0:1773941565.888261:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941565.888265:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000800:00000200:0.0:1773941565.888266:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1773941565.888270:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941565.888270:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941565.888273:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 10000000:00000001:1.0:1773941565.888276:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941565.888277:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000010:0.0:1773941565.888277:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0db600. 00000100:00000001:2.0:1773941565.888281:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941565.888284:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 10000000:01000000:1.0:1773941565.888284:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000400:00000200:0.0:1773941565.888286:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1773941565.888289:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941565.888291:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000400:00000200:0.0:1773941565.888296:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 10000000:00000010:1.0:1773941565.888299:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d04d679c0. 00000400:00000200:0.0:1773941565.888300:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045165d8 00000400:00000010:0.0:1773941565.888304:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045165d8. 00000020:00000010:1.0:1773941565.888308:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d181fc200. 00000100:00000001:0.0:1773941565.888309:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941565.888314:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c3b80 x1860112291490176/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941575 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1773941565.888317:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d181fd800. 00000040:00000010:1.0:1773941565.888323:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d181fd600. 00000100:00000001:0.0:1773941565.888326:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000020:00000010:1.0:1773941565.888330:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d18b18000. 00000100:00000040:0.0:1773941565.888330:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c3b80 x1860112291490176/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941575 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1773941565.888338:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d181fc800. 00000100:00000001:0.0:1773941565.888340:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941565.888343:0:2393:0:(events.c:68:request_out_callback()) Process leaving 80000000:00000010:1.0:1773941565.888344:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d054f31e0. 00040000:00000010:1.0:1773941565.888349:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d181fce00. 20000000:00000010:1.0:1773941565.888354:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d181fd000. 10000000:00000040:1.0:1773941565.888359:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:1.0:1773941565.888361:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941565.888366:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d181fd800. 00000040:00000010:1.0:1773941565.888370:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d181fd600. 00000020:00000010:1.0:1773941565.888372:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941565.888375:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d181fc800. 80000000:00000010:1.0:1773941565.888378:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d054f31e0. 00040000:00000010:1.0:1773941565.888381:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d181fce00. 20000000:00000010:1.0:1773941565.888384:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d181fd000. 00000020:00000010:1.0:1773941565.888386:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d181fc200. 10000000:00000010:1.0:1773941565.888388:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d04d679c0. 10000000:01000000:1.0:1773941565.888390:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941565.888399:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941565.888401:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941565.888403:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941565.888404:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941565.888406:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941565.888408:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941565.890591:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941565.890601:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941565.890603:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941565.890605:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941565.890608:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941565.890612:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941565.890617:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941565.890622:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941565.890632:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941565.890635:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324606980 00000400:00000200:0.0:1773941565.890638:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941565.890641:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xf29 [1] + 0 00000800:00000001:0.0:1773941565.890645:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941565.890651:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941565.890653:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941565.890656:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941565.890658:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941565.890662:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c3b80 x1860112291490176/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941575 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941565.890672:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c3b80 x1860112291490176/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941575 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941565.890688:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941565.890694:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941565.890699:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941565.890723:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941565.890725:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941565.890728:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941565.890729:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941565.890731:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941565.890735:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941565.890736:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941565.890738:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941565.890743:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c3b80 x1860112291490176/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941575 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941565.890750:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c3b80 x1860112291490176/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941575 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941565.890755:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941565.890757:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941565.890758:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941565.890759:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941565.890760:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941565.890762:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941565.890763:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941565.890766:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=66 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941565.890774:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941565.890784:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941565.890786:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941565.890788:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941565.890790:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941565.890792:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941565.890796:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c3b80 x1860112291490176/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941575 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941565.890802:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941565.890804:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941565.890805:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941565.890807:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941565.890811:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941565.890813:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941565.890814:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941565.890816:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941565.890817:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941565.890819:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941565.890821:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:2.0:1773941565.890825:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941565 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941565.890827:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941565.890834:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:2.0:1773941565.890835:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941565.890836:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941565.890838:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941565.890839:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941565.890842:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:2.0:1773941565.890844:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941565.890846:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941565.890849:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941565.890853:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:2.0:1773941565.890855:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941565.891259:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941565.891268:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941565.891274:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c3b80 x1860112291490176/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941575 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941565.891281:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291490176:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:0.0:1773941565.891284:0:14773:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941565.891285:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941565.891289:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941565.891291:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c3b80 x1860112291490176/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941575 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1773941565.891296:0:14773:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00000001:2.0:1773941565.891303:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00100000:0.0:1773941565.891303:0:14773:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:0.0:1773941565.891307:0:14773:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:2.0:1773941565.891308:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 10000000:01000000:0.0:1773941565.891309:0:14773:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000010:2.0:1773941565.891312:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a4400. 00000001:00000001:0.0:1773941565.891313:0:14773:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941565.891317:0:14773:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941565.891319:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:0.0:1773941565.891321:0:14773:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941565.891323:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:0.0:1773941565.891324:0:14773:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000040:2.0:1773941565.891327:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000001:00000001:0.0:1773941565.891327:0:14773:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941565.891330:0:14773:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941565.891334:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 10000000:00000001:0.0:1773941565.891334:0:14773:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941565.891338:0:14773:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000010:2.0:1773941565.891339:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d1118a000. 00000100:00080000:0.0:1773941565.891342:0:14773:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 02000000:00000001:2.0:1773941565.891345:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:0.0:1773941565.891345:0:14773:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:0.0:1773941565.891348:0:14773:0:(recover.c:147:ptlrpc_resend()) Process entered 02000000:00000001:2.0:1773941565.891349:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1773941565.891351:0:14773:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941565.891353:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c3b80. 00000100:00000001:2.0:1773941565.891358:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00080000:0.0:1773941565.891360:0:14773:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941565.891361:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941565.891364:0:14773:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:2.0:1773941565.891367:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:01000000:0.0:1773941565.891367:0:14773:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941565.891370:0:14773:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00000001:2.0:1773941565.891375:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941565.891377:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941565.891379:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941565.891380:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941565.891391:0:14773:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941565.891393:0:14773:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941565.891395:0:14773:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941565.891398:0:14773:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:0.0:1773941565.891402:0:14773:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941565.891405:0:14773:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941565.891406:0:14773:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:0.0:1773941565.891408:0:14773:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941565.891410:0:14773:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941565.894997:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d04dd4e00. 00010000:00000010:1.1:1773941565.895010:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d01488e00. 00000020:00000001:2.0:1773941566.459147:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941566.459172:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941566.459176:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941566.715088:0:461:0:(pinger.c:229:ptlrpc_pinger_process_import()) c1c1ff21-5f68-4bf6-bdc4-258644b24465->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1773941566.715100:0:461:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1773941566.715113:0:461:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c2300. 00000020:00000040:1.0:1773941566.715120:0:461:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941566.715126:0:461:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941566.715129:0:461:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941566.715135:0:461:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941566.715144:0:461:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff988d02cfd900. 00000100:00000001:1.0:1773941566.715152:0:461:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941566.715166:0:461:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging c1c1ff21-5f68-4bf6-bdc4-258644b24465->MGS req@ffff988d203c2300 x1860112291490304/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941566.715176:0:461:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941566.715181:0:461:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941566.715187:0:461:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093c3ab5c] to pc [ptlrpcd_00_00+0] req@ffff988d203c2300 x1860112291490304/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941566.715215:0:461:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941566.715220:0:461:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (294) 00000100:00000001:1.0:1773941566.715231:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1773941566.715236:0:2402:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:0.0:1773941566.715236:0:2401:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941566.715241:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941566.715242:0:2401:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941566.715250:0:2402:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941566.715254:0:2402:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941566.715256:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941566.715258:0:2402:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941566.715261:0:2402:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941566.715266:0:2402:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c2300 x1860112291490304/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941566.715273:0:2402:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941566.715292:0:2402:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941566.715297:0:2402:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941566.715300:0:2402:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941566.715304:0:2402:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941566.715309:0:2402:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2402:1860112291490304:192.168.204.114@tcp:400:kworker.0 00000100:00000001:1.0:1773941566.715316:0:2402:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941566.715323:0:2402:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941566.715327:0:2402:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941566.715330:0:2402:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941566.715336:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff988d181fc200. 02000000:00000001:1.0:1773941566.715339:0:2402:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941566.715350:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941566.715356:0:2402:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c5d8. 00000400:00000010:1.0:1773941566.715366:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000100:00000200:1.0:1773941566.715375:0:2402:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112291490304, portal 25 00000100:00000001:1.0:1773941566.715378:0:2402:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941566.715380:0:2402:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631463680 : -113743078087936 : ffff988d203c2300) 00000100:00000040:1.0:1773941566.715388:0:2402:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c2300 x1860112291490304/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941582 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941566.715395:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941566.715397:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941566.715400:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de880. 00000100:00000200:1.0:1773941566.715403:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1860112291490304, offset 0 00000400:00000200:1.0:1773941566.715410:0:2402:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941566.715419:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941566.715428:0:2402:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941566.715433:0:2402:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941566.715437:0:2402:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941566.715441:0:2402:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941566.715443:0:2402:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941566.715446:0:2402:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[108, 107] 00000400:00000200:1.0:1773941566.715449:0:2402:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:1.0:1773941566.715453:0:2402:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [219:219:256:4294967295] 192.168.204.124@tcp LPNI seq info [109:109:8:4294967295] 00000400:00000200:1.0:1773941566.715463:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941566.715473:0:2402:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:1.0:1773941566.715479:0:2402:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:1.0:1773941566.715483:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfd200. 00000800:00000200:1.0:1773941566.715490:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:1.0:1773941566.715499:0:2402:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:1.0:1773941566.715503:0:2402:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfd200 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1773941566.715517:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941566.715521:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941566.715524:0:2402:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294958311 00000100:00000001:1.0:1773941566.715533:0:2402:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294957484 00000100:00000001:1.0:1773941566.715537:0:2402:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294958311 00000100:00000001:1.0:1773941566.715541:0:2402:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294957484 00000100:00000001:1.0:1773941566.715544:0:2402:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941566.715547:0:2402:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941566.715550:0:2402:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941566.715552:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941566.715629:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941566.715634:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfd200. 00000400:00000200:0.0:1773941566.715639:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941566.715645:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941566.715651:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de880 00000400:00000010:0.0:1773941566.715654:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de880. 00000100:00000001:0.0:1773941566.715658:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941566.715662:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c2300 x1860112291490304/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941582 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941566.715672:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941566.715675:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c2300 x1860112291490304/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941582 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941566.715682:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941566.715684:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941566.716744:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941566.716757:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941566.716760:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941566.716762:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941566.716768:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941566.716772:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941566.716788:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941566.716792:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941566.716797:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941566.716801:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324606a00 00000400:00000200:0.0:1773941566.716805:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941566.716809:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xf31 [1] + 224 00000800:00000001:0.0:1773941566.716816:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941566.716825:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941566.716827:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941566.716830:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941566.716834:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941566.716839:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c2300 x1860112291490304/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941582 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941566.716849:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff988d203c2300 x1860112291490304/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941582 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941566.716865:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941566.716872:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941566.716875:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941566.716898:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941566.716901:0:2402:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941566.716906:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941566.716908:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941566.716910:0:2402:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:1.0:1773941566.716913:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:1.0:1773941566.716915:0:2402:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:1.0:1773941566.716917:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941566.716922:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c2300 x1860112291490304/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941582 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941566.716947:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c2300 x1860112291490304/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941582 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941566.716953:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941566.716955:0:2402:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941566.716998:0:2402:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941566.717002:0:2402:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941566.717004:0:2402:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941566.717006:0:2402:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941566.717008:0:2402:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941566.717016:0:2402:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=67 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941566.717018:0:2402:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941566.717021:0:2402:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1773941566.717023:0:2402:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941566.717026:0:2402:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941566.717029:0:2402:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941566.717033:0:2402:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941566.717037:0:2402:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941566.717040:0:2402:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941566.717043:0:2402:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941566.717044:0:2402:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941566.717046:0:2402:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941566.717049:0:2402:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d203c2300 x1860112291490304/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941582 ref 1 fl Rpc:RNQU/200/0 rc 0/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941566.717055:0:2402:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941566.717056:0:2402:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941566.717057:0:2402:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941566.717061:0:2402:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941566.717063:0:2402:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941566.717064:0:2402:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941566.717066:0:2402:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941566.717067:0:2402:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941566.717068:0:2402:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941566.717069:0:2402:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941566.717070:0:2402:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941566.717073:0:2402:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941566.717074:0:2402:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941566.717077:0:2402:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941566.717079:0:2402:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 288 00000400:00000200:1.0:1773941566.717088:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941566.717092:0:2402:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941566.717096:0:2402:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941566.717098:0:2402:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941566.717101:0:2402:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941566.717103:0:2402:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941566.717106:0:2402:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941566.717108:0:2402:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941566.717110:0:2402:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941566.717113:0:2402:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941566.717115:0:2402:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941566.717117:0:2402:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941566.717119:0:2402:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941566.717121:0:2402:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941566.717122:0:2402:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941566.717124:0:2402:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941566.717127:0:2402:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941566.717130:0:2402:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941566.717132:0:2402:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941566.717134:0:2402:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941566.717136:0:2402:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941566.717138:0:2402:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941566.717141:0:2402:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941566.717143:0:2402:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941566.717146:0:2402:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c3480. 00000020:00000040:1.0:1773941566.717147:0:2402:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941566.717150:0:2402:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941566.717152:0:2402:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941566.717154:0:2402:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941566.717156:0:2402:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941566.717158:0:2402:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941566.717162:0:2402:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d11188800. 00000100:00000001:1.0:1773941566.717168:0:2402:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941566.717175:0:2402:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c3480 x1860112291490432/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941566.717185:0:2402:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941566.717188:0:2402:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941566.717192:0:2402:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000099ddc7e5] to pc [ptlrpcd_rcv+-1] req@ffff988d203c3480 x1860112291490432/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941566.717215:0:2402:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941566.717219:0:2402:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941566.717221:0:2402:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:1.0:1773941566.717227:0:2402:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c2300 x1860112291490304/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941582 ref 1 fl Rpc:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941566.717235:0:2402:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941566.717239:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941566.717242:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941566.717248:0:2402:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c2300 x1860112291490304/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941582 ref 1 fl Interpret:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941566.717258:0:2402:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2402:1860112291490304:192.168.204.114@tcp:400:kworker.0 00000100:00000001:1.0:1773941566.717264:0:2402:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941566.717265:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941566.717267:0:2402:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941566.717272:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1773941566.717275:0:2402:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c2300 x1860112291490304/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941582 ref 1 fl Complete:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941566.717276:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941566.717286:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c3480 x1860112291490432/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941566.717292:0:2402:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941566.717298:0:2402:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941566.717302:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff988d181fc200. 02000000:00000001:1.0:1773941566.717306:0:2402:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941566.717310:0:2402:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941566.717313:0:2402:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941566.717320:0:2402:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941566.717324:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941566.717326:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941566.717333:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff988d02cfd900. 02000000:00000001:2.0:1773941566.717334:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941566.717339:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941566.717341:0:2402:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941566.717346:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941566.717347:0:2402:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941566.717351:0:2402:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c2300. 00000100:00000001:1.0:1773941566.717358:0:2402:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00100000:2.0:1773941566.717362:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291490432:192.168.204.114@tcp:250:ptlrpcd_00_01.0 00000100:00000001:1.0:1773941566.717362:0:2402:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941566.717368:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941566.717371:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941566.717374:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1773941566.717375:0:2402:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:2.0:1773941566.717376:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941566.717380:0:2402:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941566.717381:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1773941566.717386:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:2.0:1773941566.717387:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a4400. 00000100:00000001:1.0:1773941566.717389:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941566.717391:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941566.717397:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941566.717401:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000400:00000010:2.0:1773941566.717405:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000200:2.0:1773941566.717409:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291490432, portal 25 00000100:00000001:2.0:1773941566.717411:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941566.717413:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631468160 : -113743078083456 : ffff988d203c3480) 00000100:00000040:2.0:1773941566.717419:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c3480 x1860112291490432/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941576 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941566.717429:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941566.717433:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:2.0:1773941566.717436:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516990. 00000100:00000200:2.0:1773941566.717440:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291490432, offset 0 00000400:00000200:2.0:1773941566.717446:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941566.717452:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941566.717457:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941566.717461:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941566.717465:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941566.717468:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941566.717471:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941566.717484:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[108, 109] 00000400:00000200:2.0:1773941566.717489:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:2.0:1773941566.717493:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [220:220:256:4294967295] 192.168.204.114@tcp LPNI seq info [110:110:8:4294967295] 00000400:00000200:2.0:1773941566.717497:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941566.717503:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:2.0:1773941566.717507:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:2.0:1773941566.717513:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0db600. 00000800:00000200:2.0:1773941566.717517:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:2.0:1773941566.717525:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:2.0:1773941566.717529:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0db600 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:2.0:1773941566.717545:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941566.717550:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941566.717553:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294958313 00000100:00000001:2.0:1773941566.717558:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294958311 00000100:00000001:2.0:1773941566.717562:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294958313 00000100:00000001:2.0:1773941566.717566:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294958311 00000100:00000001:2.0:1773941566.717571:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941566.717574:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941566.717578:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941566.717581:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941566.717587:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941566.717590:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941566.717593:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941566.717594:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941566.717596:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941566.717599:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941566.717640:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941566.717645:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0db600. 00000400:00000200:0.0:1773941566.717649:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941566.717654:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941566.717657:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516990 00000400:00000010:0.0:1773941566.717658:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516990. 00000100:00000001:0.0:1773941566.717661:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941566.717664:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c3480 x1860112291490432/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941576 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941566.717671:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941566.717673:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c3480 x1860112291490432/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941576 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941566.717678:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941566.717686:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941566.719279:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941566.719290:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941566.719293:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941566.719297:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941566.719301:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941566.719307:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941566.719314:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941566.719319:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941566.719324:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941566.719328:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324606a80 00000400:00000200:0.0:1773941566.719332:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941566.719337:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xf41 [1] + 0 00000800:00000001:0.0:1773941566.719342:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941566.719350:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941566.719352:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941566.719355:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941566.719358:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941566.719362:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c3480 x1860112291490432/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941576 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941566.719370:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c3480 x1860112291490432/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941576 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941566.719384:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941566.719388:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941566.719390:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941566.719411:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941566.719413:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941566.719416:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941566.719417:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941566.719419:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941566.719424:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941566.719432:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941566.719435:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941566.719441:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c3480 x1860112291490432/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941576 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941566.719452:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c3480 x1860112291490432/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941576 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941566.719461:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941566.719463:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941566.719465:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941566.719467:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941566.719469:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941566.719471:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941566.719473:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941566.719477:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=67 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941566.719480:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941566.719482:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941566.719483:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941566.719486:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941566.719488:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941566.719490:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941566.719493:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c3480 x1860112291490432/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941576 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941566.719498:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941566.719499:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941566.719500:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941566.719503:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941566.719508:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941566.719509:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941566.719510:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941566.719511:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941566.719513:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941566.719515:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941566.719517:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:2.0:1773941566.719521:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941566 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941566.719530:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941566.719537:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:2.0:1773941566.719540:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941566.719542:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941566.719544:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941566.719545:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941566.719550:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:2.0:1773941566.719552:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941566.719555:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941566.719559:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941566.719563:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:2.0:1773941566.719566:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941566.719866:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941566.719870:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941566.719877:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c3480 x1860112291490432/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941576 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941566.719889:0:14774:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:2.0:1773941566.719890:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291490432:192.168.204.114@tcp:250:ptlrpcd_00_01.0 00000100:00000001:2.0:1773941566.719898:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941566.719901:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941566.719906:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c3480 x1860112291490432/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941576 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1773941566.719907:0:14774:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:0.0:1773941566.719915:0:14774:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:0.0:1773941566.719922:0:14774:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:2.0:1773941566.719925:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941566.719939:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941566.719942:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a4400. 10000000:01000000:0.0:1773941566.719942:0:14774:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:0.0:1773941566.719956:0:14774:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:2.0:1773941566.719957:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:0.0:1773941566.719991:0:14774:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941566.719995:0:14774:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1773941566.720003:0:14774:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941566.720004:0:14774:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941566.720011:0:14774:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941566.720013:0:14774:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941566.720016:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:0.0:1773941566.720017:0:14774:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000040:2.0:1773941566.720023:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00080000:0.0:1773941566.720025:0:14774:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000020:00000001:2.0:1773941566.720026:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941566.720031:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d11188800. 00000100:00000001:0.0:1773941566.720033:0:14774:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:0.0:1773941566.720041:0:14774:0:(recover.c:147:ptlrpc_resend()) Process entered 02000000:00000001:2.0:1773941566.720042:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941566.720048:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1773941566.720048:0:14774:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941566.720055:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c3480. 00000100:00080000:0.0:1773941566.720056:0:14774:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:0.0:1773941566.720059:0:14774:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941566.720061:0:14774:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:0.0:1773941566.720064:0:14774:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00000001:2.0:1773941566.720071:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941566.720073:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941566.720075:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941566.720079:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941566.720080:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941566.720082:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1773941566.720084:0:14774:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:00000001:2.0:1773941566.720085:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941566.720088:0:14774:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941566.720091:0:14774:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941566.720097:0:14774:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:0.0:1773941566.720106:0:14774:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941566.720109:0:14774:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941566.720110:0:14774:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:0.0:1773941566.720114:0:14774:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941566.720117:0:14774:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941567.091116:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941567.091125:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941567.091131:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941567.091133:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941567.091135:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941567.091136:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941567.091140:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941567.091142:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941567.091144:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:1.0:1773941567.091147:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941567.091149:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941567.091164:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c2300. 00000020:00000040:1.0:1773941567.091169:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941567.091185:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941567.091188:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941567.091196:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941567.091206:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d181fc200. 00000100:00000001:1.0:1773941567.091214:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941567.091219:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941567.091222:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941567.091232:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d04dd4e00. 00010000:00000001:1.0:1773941567.091239:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941567.091242:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d01488e00. 00000020:00000001:1.0:1773941567.091247:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941567.091249:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d01488e00 with handle 0x8e86767622e2a0f to hash 00000020:00000001:1.0:1773941567.091251:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941567.091276:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330112183808 : -113743597367808 : ffff988d01488e00) 00010000:00000001:1.0:1773941567.091279:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330112183808 : -113743597367808 : ffff988d01488e00) 00010000:00000001:1.0:1773941567.091283:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941567.091287:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941567.091291:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a0f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941567.091301:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a0f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941567.091313:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a0f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941567.091319:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941567.091320:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941567.091327:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941567.091329:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330528610752 : -113743180940864 : ffff988d1a1ab9c0) 00000100:00000001:1.0:1773941567.091331:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941567.091333:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631463680 : -113743078087936 : ffff988d203c2300) 00000100:00000001:1.0:1773941567.091339:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941567.091344:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941567.091346:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941567.091348:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941567.091360:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c2300 x1860112291490560/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941567.091368:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941567.091370:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941567.091373:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941567.091376:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941567.091378:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941567.091381:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c2300 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291490560:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941567.091384:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941567.091393:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941567.091396:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941567.091398:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941567.091414:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d1118bc00. 02000000:00000001:1.0:1773941567.091416:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941567.091425:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941567.091431:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c5d8. 00000400:00000010:1.0:1773941567.091441:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000100:00000200:1.0:1773941567.091451:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291490560, portal 25 00000100:00000001:1.0:1773941567.091454:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941567.091455:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631463680 : -113743078087936 : ffff988d203c2300) 00000100:00000040:1.0:1773941567.091462:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c2300 x1860112291490560/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941583 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941567.091478:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941567.091482:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941567.091486:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de660. 00000100:00000200:1.0:1773941567.091489:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291490560, offset 0 00000400:00000200:1.0:1773941567.091499:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941567.091510:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941567.091519:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941567.091525:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941567.091529:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941567.091533:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941567.091535:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941567.091538:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[110, 109] 00000400:00000200:1.0:1773941567.091541:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:1.0:1773941567.091556:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [221:221:256:4294967295] 192.168.204.124@tcp LPNI seq info [111:111:8:4294967295] 00000400:00000200:1.0:1773941567.091560:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941567.091589:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:1.0:1773941567.091595:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:1.0:1773941567.091601:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfd900. 00000800:00000200:1.0:1773941567.091609:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:1.0:1773941567.091651:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:1.0:1773941567.091656:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfd900 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941567.091676:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941567.091679:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941567.091682:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294958687 00000100:00000001:1.0:1773941567.091691:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294958313 00000100:00000001:1.0:1773941567.091714:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294958687 00000100:00000001:1.0:1773941567.091718:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294958313 00000100:00000001:1.0:1773941567.091721:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941567.091724:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941567.091726:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941567.091729:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941567.091732:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 11 seconds 00000100:00000001:1.0:1773941567.091736:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941567.091740:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941567.091871:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941567.091880:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfd900. 00000400:00000200:0.0:1773941567.091887:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941567.091897:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941567.091904:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de660 00000400:00000010:0.0:1773941567.091907:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de660. 00000100:00000001:0.0:1773941567.091911:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941567.091920:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c2300 x1860112291490560/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941583 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941567.091950:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941567.091954:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d203c2300 x1860112291490560/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941583 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941567.092008:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941567.092023:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941567.093518:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941567.093531:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941567.093534:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941567.093536:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941567.093551:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941567.093557:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941567.093567:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941567.093572:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941567.093577:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941567.093608:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324606b00 00000400:00000200:0.0:1773941567.093612:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941567.093617:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xf49 [1] + 224 00000800:00000001:0.0:1773941567.093625:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941567.093632:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941567.093634:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941567.093639:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941567.093644:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941567.093667:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c2300 x1860112291490560/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941583 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941567.093687:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d203c2300 x1860112291490560/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941583 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941567.093706:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941567.093715:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941567.093719:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941567.093772:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941567.093806:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941567.093809:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941567.093812:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:1.0:1773941567.093818:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:1.0:1773941567.093820:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:1.0:1773941567.093824:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941567.093831:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c2300 x1860112291490560/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941583 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941567.093842:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c2300 x1860112291490560/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941583 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941567.093850:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941567.093852:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941567.093854:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941567.093857:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941567.093860:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941567.093862:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941567.093865:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941567.093881:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=68 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941567.093884:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941567.093887:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941567.093891:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:1.0:1773941567.093894:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941567.093897:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941567.093900:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941567.093905:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941567.093909:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941567.093913:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941567.093915:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941567.093917:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941567.093919:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941567.093939:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d203c2300 x1860112291490560/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941583 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941567.093957:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941567.093958:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941567.093992:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941567.093995:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941567.093997:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941567.093998:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941567.094000:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941567.094001:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941567.094002:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941567.094003:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941567.094004:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941567.094007:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941567.094009:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941567.094011:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941567.094013:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 289 00000400:00000200:1.0:1773941567.094018:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941567.094021:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941567.094024:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941567.094031:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941567.094034:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941567.094036:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941567.094039:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941567.094040:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941567.094042:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941567.094044:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941567.094046:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941567.094048:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941567.094050:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941567.094051:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941567.094052:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941567.094053:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941567.094056:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941567.094058:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941567.094060:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941567.094062:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941567.094064:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941567.094066:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941567.094069:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941567.094070:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941567.094073:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c2a00. 00000020:00000040:1.0:1773941567.094075:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941567.094077:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941567.094078:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941567.094080:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941567.094081:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941567.094083:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941567.094086:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d11189400. 00000100:00000001:1.0:1773941567.094089:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941567.094095:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c2a00 x1860112291490688/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941567.094108:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941567.094110:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941567.094114:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fb7523b4] to pc [ptlrpcd_rcv+-1] req@ffff988d203c2a00 x1860112291490688/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941567.094141:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941567.094143:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941567.094145:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941567.094147:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941567.094149:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941567.094150:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941567.094151:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941567.094153:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941567.094155:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941567.094159:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c2300 x1860112291490560/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941583 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941567.094164:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941567.094168:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941567.094168:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941567.094171:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941567.094174:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941567.094175:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941567.094178:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941567.094179:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c2300 x1860112291490560/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941583 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941567.094187:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c2a00 x1860112291490688/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941567.094190:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c2300 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291490560:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941567.094199:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941567.094204:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941567.094205:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941567.094207:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:2.0:1773941567.094208:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941567.094210:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1773941567.094212:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941567.094216:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000040:1.0:1773941567.094216:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c2300 x1860112291490560/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941583 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1773941567.094222:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941567.094229:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291490688:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941567.094231:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941567.094233:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941567.094236:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941567.094239:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941567.094239:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1a1ab9c0. 02000000:00000001:2.0:1773941567.094244:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1773941567.094245:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941567.094248:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000010:2.0:1773941567.094249:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a4400. 00010000:00000001:1.0:1773941567.094251:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941567.094255:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941567.094256:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941567.094258:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488e00 refcount=4 00000020:00000001:1.0:1773941567.094259:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112183808 : -113743597367808 : ffff988d01488e00) 02000000:00000001:2.0:1773941567.094263:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941567.094264:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112183808 : -113743597367808 : ffff988d01488e00) 00000400:00000200:2.0:1773941567.094270:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:1.0:1773941567.094270:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a0f lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941567.094277:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00010000:00000001:1.0:1773941567.094279:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000010:2.0:1773941567.094282:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00010000:00010000:1.0:1773941567.094284:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a0f lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:2.0:1773941567.094287:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291490688, portal 25 00000100:00000001:2.0:1773941567.094292:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00000001:1.0:1773941567.094293:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000001:2.0:1773941567.094294:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631465472 : -113743078086144 : ffff988d203c2a00) 00010000:00010000:1.0:1773941567.094297:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a0f lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941567.094302:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c2a00 x1860112291490688/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941577 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941567.094307:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941567.094310:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941567.094313:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a0f lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941567.094314:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941567.094320:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00000001:1.0:1773941567.094323:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941567.094325:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941567.094327:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941567.094329:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a0f lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941567.094333:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045166e8. 00010000:00010000:1.0:1773941567.094336:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d01488e00 already unused, calling callback (ffffffffc12b4c20) 00000100:00000200:2.0:1773941567.094339:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291490688, offset 0 00000400:00000200:2.0:1773941567.094345:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 10000000:00000001:1.0:1773941567.094349:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:2.0:1773941567.094352:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 10000000:00010000:1.0:1773941567.094353:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a0f lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941567.094359:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:1.0:1773941567.094362:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000400:00000200:2.0:1773941567.094365:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:1.0:1773941567.094365:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941567.094369:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:2.0:1773941567.094371:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000020:00000040:1.0:1773941567.094372:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488e00 refcount=5 00000400:00000200:2.0:1773941567.094376:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000020:00000001:1.0:1773941567.094376:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112183808 : -113743597367808 : ffff988d01488e00) 00010000:00000001:1.0:1773941567.094380:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112183808 : -113743597367808 : ffff988d01488e00) 00000400:00000200:2.0:1773941567.094382:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941567.094385:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:2.0:1773941567.094387:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[110, 111] 00010000:00010000:1.0:1773941567.094389:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a0f lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941567.094394:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:2.0:1773941567.094401:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [222:222:256:4294967295] 192.168.204.114@tcp LPNI seq info [112:112:8:4294967295] 10000000:00000001:1.0:1773941567.094401:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941567.094404:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a0f lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941567.094408:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 10000000:01000000:1.0:1773941567.094415:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941567.094420:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000400:00000200:2.0:1773941567.094422:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 10000000:00000001:1.0:1773941567.094423:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941567.094430:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000800:00000200:2.0:1773941567.094431:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00010000:00000001:1.0:1773941567.094434:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000800:00000010:2.0:1773941567.094437:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0db900. 00010000:00000001:1.0:1773941567.094440:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000800:00000200:2.0:1773941567.094444:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00010000:00000001:1.0:1773941567.094444:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941567.094449:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941567.094453:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941567.094455:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000020:00000040:1.0:1773941567.094459:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d01488e00 with handle 0x8e86767622e2a0f from hash 00000800:00000200:2.0:1773941567.094460:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0db900 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941567.094466:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941567.094469:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941567.094471:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941567.094472:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000100:00000001:2.0:1773941567.094474:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941567.094475:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941567.094480:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000400:00000200:2.0:1773941567.094481:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941567.094485:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294958690 00010000:00000001:1.0:1773941567.094487:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941567.094490:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294958687 00010000:00000001:1.0:1773941567.094495:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941567.094497:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294958690 00010000:00000001:1.0:1773941567.094498:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941567.094512:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294958687 10000000:00000001:1.0:1773941567.094513:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941567.094519:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00010000:1.0:1773941567.094519:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a0f lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941567.094524:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941567.094530:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941567.094531:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941567.094534:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941567.094536:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:1.0:1773941567.094537:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941567.094542:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941567.094544:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941567.094547:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:1.0:1773941567.094547:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941567.094548:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941567.094551:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a0f lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941567.094557:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00010000:00000040:1.0:1773941567.094558:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d04dd4e00 count: 0 00000100:00000001:2.0:1773941567.094561:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941567.094564:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941567.094569:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941567.094572:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941567.094593:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941567.094595:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941567.094598:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941567.094605:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c2300 x1860112291490560/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941583 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941567.094611:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941567.094614:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941567.094617:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d1118bc00. 00000800:00000200:0.0:1773941567.094620:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 02000000:00000001:1.0:1773941567.094621:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941567.094627:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000800:00000010:0.0:1773941567.094628:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0db900. 00000020:00000040:1.0:1773941567.094630:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941567.094636:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 00000400:00000200:0.0:1773941567.094639:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000010:1.0:1773941567.094640:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d181fc200. 02000000:00000001:1.0:1773941567.094668:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941567.094681:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941567.094684:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c2300. 00000400:00000200:0.0:1773941567.094692:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:1.0:1773941567.094694:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941567.094698:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941567.094701:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045166e8 10000000:00000001:1.0:1773941567.094702:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000010:0.0:1773941567.094706:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045166e8. 00000100:00000001:0.0:1773941567.094711:0:2393:0:(events.c:34:request_out_callback()) Process entered 10000000:01000000:1.0:1773941567.094715:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941567.094720:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000100:00000200:0.0:1773941567.094721:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c2a00 x1860112291490688/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941577 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000010:1.0:1773941567.094726:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d04d679c0. 00000020:00000010:1.0:1773941567.094733:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d181fc200. 00000020:00000010:1.0:1773941567.094750:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d181fd000. 00000100:00000001:0.0:1773941567.094755:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000040:00000010:1.0:1773941567.094757:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d181fce00. 00000100:00000040:0.0:1773941567.094764:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c2a00 x1860112291490688/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941577 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1773941567.094765:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941567.094780:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d181fc800. 00000100:00000001:0.0:1773941567.094793:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 80000000:00000010:1.0:1773941567.094794:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d054f31e0. 00000100:00000001:0.0:1773941567.094807:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00040000:00000010:1.0:1773941567.094810:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d181fd600. 20000000:00000010:1.0:1773941567.094818:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d181fd800. 10000000:00000040:1.0:1773941567.094824:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:1.0:1773941567.094827:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941567.094833:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d181fd000. 00000040:00000010:1.0:1773941567.094851:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d181fce00. 00000020:00000010:1.0:1773941567.094856:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941567.094860:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d181fc800. 80000000:00000010:1.0:1773941567.094864:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d054f31e0. 00040000:00000010:1.0:1773941567.094869:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d181fd600. 20000000:00000010:1.0:1773941567.094871:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d181fd800. 00000020:00000010:1.0:1773941567.094874:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d181fc200. 10000000:00000010:1.0:1773941567.094876:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d04d679c0. 10000000:01000000:1.0:1773941567.094879:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941567.094882:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941567.094884:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941567.094886:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941567.094887:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941567.094889:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941567.094891:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941567.096128:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941567.096137:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941567.096140:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941567.096142:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941567.096147:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941567.096151:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941567.096156:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941567.096159:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941567.096163:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941567.096166:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324606b80 00000400:00000200:0.0:1773941567.096170:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941567.096174:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xf59 [1] + 0 00000800:00000001:0.0:1773941567.096179:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941567.096185:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941567.096187:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941567.096190:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941567.096193:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941567.096197:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c2a00 x1860112291490688/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941577 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941567.096207:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c2a00 x1860112291490688/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941577 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941567.096224:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941567.096229:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941567.096231:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941567.096356:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941567.096359:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941567.096363:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941567.096364:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941567.096366:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941567.096370:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941567.096372:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941567.096374:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941567.096380:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c2a00 x1860112291490688/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941577 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941567.096388:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c2a00 x1860112291490688/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941577 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941567.096394:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941567.096395:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941567.096396:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941567.096398:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941567.096399:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941567.096400:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941567.096402:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941567.096421:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=68 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941567.096425:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941567.096428:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941567.096430:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941567.096433:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941567.096436:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941567.096439:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941567.096444:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c2a00 x1860112291490688/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941577 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941567.096453:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941567.096455:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941567.096486:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941567.096489:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941567.096493:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941567.096495:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941567.096497:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941567.096499:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941567.096501:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941567.096503:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941567.096505:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:2.0:1773941567.096508:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941567 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941567.096511:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941567.096518:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:2.0:1773941567.096520:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941567.096522:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941567.096524:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941567.096525:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941567.096529:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:2.0:1773941567.096532:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941567.096534:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941567.096538:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941567.096541:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:2.0:1773941567.096544:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941567.096893:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941567.096898:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941567.096906:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c2a00 x1860112291490688/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941577 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941567.096919:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291490688:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:0.0:1773941567.096936:0:14775:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941567.096960:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1773941567.097004:0:14775:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:0.0:1773941567.097009:0:14775:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:0.0:1773941567.097014:0:14775:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941567.097018:0:14775:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:0.0:1773941567.097024:0:14775:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:2.0:1773941567.097026:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000001:00000001:0.0:1773941567.097029:0:14775:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941567.097034:0:14775:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:0.0:1773941567.097037:0:14775:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000040:2.0:1773941567.097039:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c2a00 x1860112291490688/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941577 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000001:00000001:0.0:1773941567.097040:0:14775:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:0.0:1773941567.097044:0:14775:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941567.097052:0:14775:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941567.097059:0:14775:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941567.097068:0:14775:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:0.0:1773941567.097087:0:14775:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:0.0:1773941567.097090:0:14775:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:0.0:1773941567.097096:0:14775:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941567.097110:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00080000:0.0:1773941567.097110:0:14775:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:0.0:1773941567.097119:0:14775:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:2.0:1773941567.097120:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941567.097126:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a4400. 10000000:01000000:0.0:1773941567.097126:0:14775:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 02000000:00000001:2.0:1773941567.097142:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 10000000:00000040:0.0:1773941567.097142:0:14775:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000020:00000001:2.0:1773941567.097151:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941567.097154:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941567.097157:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941567.097159:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d11189400. 02000000:00000001:2.0:1773941567.097164:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941567.097165:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941567.097167:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c2a00. 00000100:00080000:0.0:1773941567.097168:0:14775:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941567.097171:0:14775:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941567.097172:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941567.097182:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941567.097183:0:14775:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941567.097193:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:02000000:0.0:1773941567.097203:0:14775:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:2.0:1773941567.097204:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941567.097210:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941567.097215:0:14775:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941567.097218:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1773941567.097220:0:14775:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:2.0:1773941567.097221:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1773941567.097230:0:14775:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:0.0:1773941567.097234:0:14775:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941567.097236:0:14775:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941567.100025:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d04dd4e00. 00010000:00000010:1.1:1773941567.100032:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d01488e00. 00000020:00000001:2.0:1773941567.483165:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941567.483175:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941567.483181:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1773941568.379088:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294958690, at 4294958720 00000100:00000010:0.0:1773941568.379105:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3660. 00000100:00000040:1.0:1773941568.379156:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294958690, at 4294958720 00000100:00000010:1.0:1773941568.379198:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3600. 00000020:00000001:2.0:1773941568.507230:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941568.507240:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941568.507246:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941568.827036:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941568.827045:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941568.827074:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941568.827077:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941568.827078:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941568.827079:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941568.827082:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941568.827084:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941568.827086:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:1.0:1773941568.827089:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941568.827090:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941568.827099:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c2300. 00000020:00000040:1.0:1773941568.827104:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941568.827109:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941568.827112:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941568.827118:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941568.827126:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d181fc200. 00000100:00000001:1.0:1773941568.827135:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941568.827140:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941568.827144:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941568.827153:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d04dd4e00. 00010000:00000001:1.0:1773941568.827161:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941568.827165:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d01488e00. 00000020:00000001:1.0:1773941568.827170:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941568.827174:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d01488e00 with handle 0x8e86767622e2a16 to hash 00000020:00000001:1.0:1773941568.827176:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941568.827178:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330112183808 : -113743597367808 : ffff988d01488e00) 00010000:00000001:1.0:1773941568.827182:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330112183808 : -113743597367808 : ffff988d01488e00) 00010000:00000001:1.0:1773941568.827187:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941568.827191:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941568.827197:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a16 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941568.827211:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a16 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941568.827222:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a16 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941568.827233:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941568.827235:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941568.827241:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941568.827243:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330528610752 : -113743180940864 : ffff988d1a1ab9c0) 00000100:00000001:1.0:1773941568.827246:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941568.827247:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631463680 : -113743078087936 : ffff988d203c2300) 00000100:00000001:1.0:1773941568.827253:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941568.827257:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941568.827259:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941568.827261:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941568.827272:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c2300 x1860112291490816/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941568.827279:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941568.827281:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941568.827285:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941568.827287:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941568.827289:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941568.827292:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c2300 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291490816:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941568.827295:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941568.827301:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941568.827304:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941568.827306:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941568.827310:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d1118bc00. 02000000:00000001:1.0:1773941568.827312:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941568.827326:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941568.827330:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c5d8. 00000400:00000010:1.0:1773941568.827337:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000100:00000200:1.0:1773941568.827344:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291490816, portal 25 00000100:00000001:1.0:1773941568.827346:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941568.827347:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631463680 : -113743078087936 : ffff988d203c2300) 00000100:00000040:1.0:1773941568.827351:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c2300 x1860112291490816/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941584 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941568.827365:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941568.827368:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941568.827370:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de000. 00000100:00000200:1.0:1773941568.827372:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291490816, offset 0 00000400:00000200:1.0:1773941568.827378:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941568.827387:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941568.827395:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941568.827400:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941568.827404:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941568.827408:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941568.827410:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941568.827412:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[112, 111] 00000400:00000200:1.0:1773941568.827415:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:1.0:1773941568.827418:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [223:223:256:4294967295] 192.168.204.124@tcp LPNI seq info [113:113:8:4294967295] 00000400:00000200:1.0:1773941568.827421:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941568.827429:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:1.0:1773941568.827434:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:1.0:1773941568.827438:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfdc00. 00000800:00000200:1.0:1773941568.827445:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:1.0:1773941568.827453:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:1.0:1773941568.827458:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfdc00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941568.827475:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941568.827479:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941568.827481:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294960423 00000100:00000010:1.0:1773941568.827485:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3600. 00000100:00000001:1.0:1773941568.827508:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1773941568.827516:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294960423 00000100:00000010:1.0:1773941568.827519:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f31e0. 00000100:00000001:1.0:1773941568.827521:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1773941568.827532:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941568.827535:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941568.827538:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941568.827540:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941568.827544:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 11 seconds 00000100:00000001:1.0:1773941568.827547:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941568.827551:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941568.827712:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941568.827720:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfdc00. 00000400:00000200:0.0:1773941568.827726:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941568.827732:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941568.827751:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de000 00000400:00000010:0.0:1773941568.827753:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de000. 00000100:00000001:0.0:1773941568.827757:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941568.827764:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c2300 x1860112291490816/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941584 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941568.827773:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941568.827776:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d203c2300 x1860112291490816/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941584 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941568.827792:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941568.827794:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941568.828751:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941568.828761:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941568.828764:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941568.828766:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941568.828782:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941568.828787:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941568.828792:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941568.828796:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941568.828800:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941568.828803:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324606c00 00000400:00000200:0.0:1773941568.828807:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941568.828811:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xf61 [1] + 224 00000800:00000001:0.0:1773941568.828817:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941568.828824:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941568.828825:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941568.828828:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941568.828832:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941568.828837:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c2300 x1860112291490816/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941584 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941568.828849:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d203c2300 x1860112291490816/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941584 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941568.828869:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941568.828887:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000100:00000001:1.0:1773941568.828918:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941568.828937:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941568.828940:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941568.828943:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:1.0:1773941568.828949:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:1.0:1773941568.828951:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:1.0:1773941568.828955:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941568.829000:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c2300 x1860112291490816/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941584 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941568.829010:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c2300 x1860112291490816/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941584 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941568.829016:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941568.829017:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941568.829020:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941568.829023:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941568.829026:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941568.829029:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941568.829032:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941568.829038:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=69 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941568.829041:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941568.829044:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941568.829048:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000800:00000001:0.0:1773941568.829063:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941568.833214:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941568.833221:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941568.833225:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941568.833230:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941568.833235:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941568.837866:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941568.837869:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941568.837870:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941568.837873:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941568.837878:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d203c2300 x1860112291490816/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941584 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941568.837945:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941568.837947:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941568.837949:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941568.837952:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941568.837955:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941568.837956:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941568.837958:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941568.837982:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941568.837984:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941568.837987:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941568.837990:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941568.837993:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941568.837996:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941568.838000:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941568.838006:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 290 00000400:00000200:1.0:1773941568.838014:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941568.838022:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941568.838027:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941568.838031:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941568.838052:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941568.838055:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941568.838059:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941568.838061:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941568.838063:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941568.838066:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941568.838069:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941568.838072:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941568.838083:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941568.838085:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941568.838088:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941568.838090:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941568.838094:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941568.838097:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941568.838100:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941568.838102:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941568.838105:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941568.838107:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941568.838109:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941568.838111:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941568.838116:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c0380. 00000020:00000040:1.0:1773941568.838118:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941568.838122:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941568.838123:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941568.838126:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941568.838128:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941568.838131:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941568.838136:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d11189c00. 00000100:00000001:1.0:1773941568.838142:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941568.838151:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c0380 x1860112291490944/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941568.838162:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941568.838165:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941568.838172:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000da1038d4] to pc [ptlrpcd_rcv+-1] req@ffff988d203c0380 x1860112291490944/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941568.838201:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941568.838204:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941568.838207:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941568.838228:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941568.838229:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941568.838233:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941568.838235:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941568.838238:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941568.838244:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941568.838246:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c0380 x1860112291490944/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941568.838248:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941568.838258:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941568.838264:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941568.838271:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941568.838273:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941568.838277:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c2300 x1860112291490816/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941584 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1773941568.838279:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941568.838286:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941568.838297:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941568.838307:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00100000:2.0:1773941568.838309:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291490944:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941568.838312:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941568.838314:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941568.838316:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941568.838325:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c2300 x1860112291490816/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941584 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941568.838327:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941568.838336:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941568.838341:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941568.838342:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c2300 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291490816:192.168.204.114@tcp:101:ll_cfg_requeue.0 02000000:00000001:2.0:1773941568.838351:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1773941568.838354:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:2.0:1773941568.838357:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a4400. 00000100:00000001:1.0:1773941568.838359:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:2.0:1773941568.838362:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941568.838364:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941568.838367:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000400:00000200:2.0:1773941568.838371:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941568.838376:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000100:00000040:1.0:1773941568.838377:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c2300 x1860112291490816/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941584 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1773941568.838389:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000001:1.0:1773941568.838390:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941568.838395:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941568.838399:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941568.838401:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000200:2.0:1773941568.838402:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291490944, portal 25 00010000:00000001:1.0:1773941568.838404:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941568.838410:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000001:2.0:1773941568.838411:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000020:00000001:1.0:1773941568.838413:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941568.838417:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488e00 refcount=4 00000100:00000001:2.0:1773941568.838418:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631455616 : -113743078096000 : ffff988d203c0380) 00000020:00000001:1.0:1773941568.838421:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112183808 : -113743597367808 : ffff988d01488e00) 00010000:00000001:1.0:1773941568.838429:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112183808 : -113743597367808 : ffff988d01488e00) 00010000:00010000:1.0:1773941568.838433:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a16 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941568.838437:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c0380 x1860112291490944/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941578 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941568.838465:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00010000:00010000:1.0:1773941568.838474:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a16 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941568.838476:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:1.0:1773941568.838483:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000040:2.0:1773941568.838484:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00010000:1.0:1773941568.838487:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a16 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941568.838489:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516bb0. 00000100:00000200:2.0:1773941568.838500:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291490944, offset 0 00010000:00000001:1.0:1773941568.838501:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941568.838506:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941568.838508:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a16 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941568.838514:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941568.838532:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941568.838533:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941568.838545:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941568.838549:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:1.0:1773941568.838553:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:2.0:1773941568.838557:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00010000:1.0:1773941568.838562:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a16 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941568.838563:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941568.838570:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941568.838578:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941568.838584:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[112, 113] 00000400:00000200:2.0:1773941568.838608:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00010000:00010000:1.0:1773941568.838608:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d01488e00 already unused, calling callback (ffffffffc12b4c20) 10000000:00000001:1.0:1773941568.838620:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:2.0:1773941568.838628:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [224:224:256:4294967295] 192.168.204.114@tcp LPNI seq info [114:114:8:4294967295] 10000000:00010000:1.0:1773941568.838629:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a16 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941568.838662:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941568.838676:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941568.838681:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:2.0:1773941568.838682:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000020:00000001:1.0:1773941568.838686:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941568.838689:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488e00 refcount=5 00000800:00000200:2.0:1773941568.838696:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000020:00000001:1.0:1773941568.838698:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112183808 : -113743597367808 : ffff988d01488e00) 00010000:00000001:1.0:1773941568.838700:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112183808 : -113743597367808 : ffff988d01488e00) 00010000:00000001:1.0:1773941568.838705:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000800:00000010:2.0:1773941568.838709:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0db000. 00010000:00010000:1.0:1773941568.838716:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a16 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941568.838717:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:2.0:1773941568.838736:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 10000000:00000001:1.0:1773941568.838736:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941568.838745:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a16 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941568.838746:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0db000 type 1, nob 616 niov 1 nkiov 1 10000000:01000000:1.0:1773941568.838765:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941568.838767:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941568.838769:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941568.838773:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000100:00000001:2.0:1773941568.838774:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941568.838776:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941568.838780:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000400:00000200:2.0:1773941568.838781:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941568.838785:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000100:00000001:2.0:1773941568.838788:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294960434 00010000:00000001:1.0:1773941568.838789:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941568.838795:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941568.838800:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d01488e00 with handle 0x8e86767622e2a16 from hash 00000100:00000001:2.0:1773941568.838801:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294960423 00010000:00000001:1.0:1773941568.838807:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000100:00000001:2.0:1773941568.838809:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294960434 00010000:00000001:1.0:1773941568.838810:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941568.838816:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941568.838819:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294960423 00010000:00000001:1.0:1773941568.838819:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941568.838826:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000100:00000001:2.0:1773941568.838827:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941568.838832:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000100:00000001:2.0:1773941568.838833:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941568.838845:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941568.838847:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941568.838850:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941568.838852:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:1.0:1773941568.838854:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941568.838870:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 10000000:00000001:1.0:1773941568.838871:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941568.838876:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941568.838879:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000800:00000010:0.0:1773941568.838879:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0db000. 00010000:00010000:1.0:1773941568.838880:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a16 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941568.838903:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:0.0:1773941568.838907:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1773941568.838911:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941568.838916:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941568.838923:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941568.838924:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00010000:00000001:1.0:1773941568.838939:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:0.0:1773941568.838940:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516bb0 00010000:00000001:1.0:1773941568.838942:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000010:0.0:1773941568.838944:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516bb0. 00010000:00000001:1.0:1773941568.838945:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941568.838952:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000100:00000001:0.0:1773941568.838954:0:2393:0:(events.c:34:request_out_callback()) Process entered 00010000:00000001:1.0:1773941568.838957:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941568.838959:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941568.838965:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941568.838969:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2a16 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:0.0:1773941568.838984:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c0380 x1860112291490944/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941578 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941568.838992:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941568.838997:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c0380 x1860112291490944/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941578 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000040:1.0:1773941568.839009:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d04dd4e00 count: 0 00000100:00000001:0.0:1773941568.839011:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941568.839016:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:1.0:1773941568.839030:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941568.839033:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941568.839036:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941568.839042:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c2300 x1860112291490816/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941584 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941568.839050:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941568.839053:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941568.839056:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d1118bc00. 02000000:00000001:1.0:1773941568.839060:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941568.839061:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941568.839063:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941568.839065:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941568.839067:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d181fc200. 02000000:00000001:1.0:1773941568.839070:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941568.839072:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941568.839073:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c2300. 00000100:00000001:1.0:1773941568.839093:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941568.839095:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941568.839097:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941568.839099:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941568.839102:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941568.839107:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d04d679c0. 00000020:00000010:1.0:1773941568.839111:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d181fc200. 00000020:00000010:1.0:1773941568.839116:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d181fd800. 00000040:00000010:1.0:1773941568.839120:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d181fd600. 00000020:00000010:1.0:1773941568.839138:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941568.839146:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d181fc800. 80000000:00000010:1.0:1773941568.839151:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d054f3960. 00040000:00000010:1.0:1773941568.839158:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d181fce00. 20000000:00000010:1.0:1773941568.839165:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d181fd000. 10000000:00000040:1.0:1773941568.839179:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:1.0:1773941568.839182:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941568.839190:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d181fd800. 00000040:00000010:1.0:1773941568.839195:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d181fd600. 00000020:00000010:1.0:1773941568.839199:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941568.839204:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d181fc800. 80000000:00000010:1.0:1773941568.839224:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d054f3960. 00040000:00000010:1.0:1773941568.839229:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d181fce00. 20000000:00000010:1.0:1773941568.839233:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d181fd000. 00000020:00000010:1.0:1773941568.839236:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d181fc200. 10000000:00000010:1.0:1773941568.839239:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d04d679c0. 10000000:01000000:1.0:1773941568.839243:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941568.839246:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941568.839248:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941568.839250:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941568.839251:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941568.839253:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941568.839255:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941568.843756:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941568.843766:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941568.843769:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941568.843772:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941568.843778:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941568.843783:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941568.843790:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941568.843793:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941568.843797:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941568.843800:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324606c80 00000400:00000200:0.0:1773941568.843805:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941568.843810:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xf71 [1] + 0 00000800:00000001:0.0:1773941568.843816:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941568.843822:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941568.843825:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941568.843829:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941568.843834:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941568.843848:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c0380 x1860112291490944/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941578 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941568.843859:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c0380 x1860112291490944/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941578 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941568.843873:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941568.843879:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941568.843882:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941568.843914:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941568.843917:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941568.843921:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941568.843922:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941568.843924:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941568.843928:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941568.843929:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941568.843931:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941568.843936:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c0380 x1860112291490944/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941578 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941568.843944:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c0380 x1860112291490944/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941578 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941568.843949:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941568.843956:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941568.843958:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941568.843983:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941568.843986:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941568.843990:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941568.843993:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941568.844000:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=69 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941568.844004:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941568.844007:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941568.844009:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941568.844012:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941568.844014:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941568.844016:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941568.844020:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c0380 x1860112291490944/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941578 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941568.844025:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941568.844028:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941568.844030:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941568.844032:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941568.844037:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941568.844040:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941568.844042:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941568.844044:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941568.844047:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941568.844049:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941568.844068:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:2.0:1773941568.844071:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941568 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941568.844073:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941568.844079:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:2.0:1773941568.844081:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941568.844085:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941568.844086:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941568.844088:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941568.844091:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:2.0:1773941568.844092:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941568.844101:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941568.844105:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941568.844108:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:2.0:1773941568.844110:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941568.844337:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941568.844340:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941568.844344:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c0380 x1860112291490944/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941578 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941568.844351:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291490944:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941568.844355:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941568.844356:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941568.844360:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c0380 x1860112291490944/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941578 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941568.844360:0:14776:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941568.844369:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941568.844373:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00080000:1.0:1773941568.844377:0:14776:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 02000000:00000010:2.0:1773941568.844380:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a4400. 00000100:00100000:1.0:1773941568.844383:0:14776:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000001:2.0:1773941568.844385:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:1.0:1773941568.844388:0:14776:0:(obd_class.h:1215:obd_import_event()) Process entered 00000020:00000001:2.0:1773941568.844389:0:2400:0:(genops.c:1236:class_import_put()) Process entered 10000000:01000000:1.0:1773941568.844393:0:14776:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000020:00000040:2.0:1773941568.844395:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941568.844402:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:1.0:1773941568.844402:0:14776:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941568.844414:0:14776:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000010:2.0:1773941568.844415:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d11189c00. 02000000:00000001:2.0:1773941568.844430:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000001:00000001:1.0:1773941568.844430:0:14776:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941568.844436:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000001:00000001:1.0:1773941568.844438:0:14776:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000010:2.0:1773941568.844441:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c0380. 00000001:00000001:1.0:1773941568.844441:0:14776:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941568.844456:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000001:00000001:1.0:1773941568.844456:0:14776:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941568.844466:0:14776:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941568.844467:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941568.844485:0:14776:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941568.844487:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941568.844497:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:1.0:1773941568.844498:0:14776:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941568.844506:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941568.844508:0:14776:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941568.844510:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941568.844512:0:14776:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:2.0:1773941568.844513:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941568.844515:0:14776:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941568.844530:0:14776:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941568.844534:0:14776:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941568.844536:0:14776:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941568.844539:0:14776:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:1.0:1773941568.844560:0:14776:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941568.844563:0:14776:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941568.844565:0:14776:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941568.844570:0:14776:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00010000:00000010:1.1:1773941568.846903:0:14776:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d04dd4e00. 00010000:00000010:1.1:1773941568.846911:0:14776:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d01488e00. 00000100:00000001:1.0:1773941568.848095:0:14776:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941568.848099:0:14776:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941568.848100:0:14776:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941568.848103:0:14776:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941568.848104:0:14776:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941569.531172:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941569.531181:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941569.531208:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941570.107070:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294960434, at 4294960448 00000100:00000010:1.0:1773941570.107089:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f31e0. 00000100:00000040:0.0:1773941570.107143:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294960434, at 4294960448 00000100:00000010:0.0:1773941570.107158:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3600. 10000000:00000001:0.0:1773941570.235157:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1773941570.235166:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941570.235173:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941570.235176:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941570.235179:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941570.235181:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1773941570.235184:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:0.0:1773941570.235187:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:0.0:1773941570.235190:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:0.0:1773941570.235193:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1773941570.235195:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:0.0:1773941570.235209:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10836680. 00000020:00000040:0.0:1773941570.235214:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941570.235221:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941570.235225:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941570.235231:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941570.235240:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065c4a00. 00000100:00000001:0.0:1773941570.235248:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941570.235253:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1773941570.235257:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1773941570.235267:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d21ec1400. 00010000:00000001:0.0:1773941570.235284:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1773941570.235288:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d00031e00. 00000020:00000001:0.0:1773941570.235293:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:0.0:1773941570.235296:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d00031e00 with handle 0x8e86767622e2a1d to hash 00000020:00000001:0.0:1773941570.235298:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:0.0:1773941570.235319:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330090855936 : -113743618695680 : ffff988d00031e00) 00010000:00000001:0.0:1773941570.235323:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330090855936 : -113743618695680 : ffff988d00031e00) 00010000:00000001:0.0:1773941570.235328:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941570.235331:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941570.235337:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2a1d lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941570.235349:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2a1d lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941570.235363:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2a1d lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941570.235370:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1773941570.235373:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941570.235380:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb6a80. 00000100:00000001:0.0:1773941570.235382:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330139634304 : -113743569917312 : ffff988d02eb6a80) 00000100:00000001:0.0:1773941570.235385:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941570.235387:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367698560 : -113743341853056 : ffff988d10836680) 00000100:00000001:0.0:1773941570.235393:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941570.235397:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941570.235400:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941570.235402:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941570.235415:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10836680 x1860112291491072/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941570.235426:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941570.235428:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941570.235433:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941570.235454:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941570.235457:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941570.235462:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10836680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291491072:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941570.235467:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941570.235479:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941570.235483:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941570.235486:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941570.235491:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d02ce7000. 02000000:00000001:0.0:1773941570.235494:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941570.235504:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941570.235509:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0014b580. 00000400:00000010:0.0:1773941570.235517:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833ee0. 00000100:00000200:0.0:1773941570.235525:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291491072, portal 25 00000100:00000001:0.0:1773941570.235528:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941570.235530:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367698560 : -113743341853056 : ffff988d10836680) 00000100:00000040:0.0:1773941570.235537:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10836680 x1860112291491072/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941586 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941570.235547:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941570.235551:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941570.235554:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d108335d8. 00000100:00000200:0.0:1773941570.235558:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291491072, offset 0 00000400:00000200:0.0:1773941570.235566:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941570.235576:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941570.235586:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941570.235592:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941570.235598:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941570.235603:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941570.235607:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941570.235611:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[114, 113] 00000400:00000200:0.0:1773941570.235627:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:0.0:1773941570.235633:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [225:225:256:4294967295] 192.168.204.124@tcp LPNI seq info [115:115:8:4294967295] 00000400:00000200:0.0:1773941570.235639:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941570.235652:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:0.0:1773941570.235660:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:0.0:1773941570.235666:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a20bd00. 00000800:00000200:0.0:1773941570.235676:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:0.0:1773941570.235693:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:0.0:1773941570.235699:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a20bd00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1773941570.235725:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941570.235730:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941570.235735:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294961831 00000100:00000010:0.0:1773941570.235740:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580b40. 00000100:00000001:0.0:1773941570.235749:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1773941570.235756:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294961831 00000100:00000010:0.0:1773941570.235761:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580000. 00000100:00000001:0.0:1773941570.235766:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1773941570.235773:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941570.235777:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941570.235781:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941570.235783:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1773941570.235787:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6a80 going to sleep for 11 seconds 00000100:00000001:0.0:1773941570.235791:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941570.235795:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941570.235946:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941570.235955:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a20bd00. 00000400:00000200:1.0:1773941570.235990:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941570.236011:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941570.236018:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d108335d8 00000400:00000010:1.0:1773941570.236021:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d108335d8. 00000100:00000001:1.0:1773941570.236027:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941570.236035:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10836680 x1860112291491072/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941586 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941570.236050:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941570.236055:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d10836680 x1860112291491072/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941586 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941570.236065:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941570.236068:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941570.237783:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941570.237802:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941570.237806:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941570.237809:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941570.237817:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941570.237824:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941570.237832:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941570.237838:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941570.237844:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941570.237849:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324606d00 00000400:00000200:0.0:1773941570.237854:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941570.237885:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xf79 [1] + 224 00000800:00000001:0.0:1773941570.237897:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941570.237909:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941570.237912:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941570.237917:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941570.237935:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941570.237943:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10836680 x1860112291491072/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941586 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941570.237958:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d10836680 x1860112291491072/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941586 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941570.238021:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941570.238033:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941570.238038:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941570.238083:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941570.238108:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941570.238112:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941570.238116:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0014b580. 00000400:00000200:1.0:1773941570.238124:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833ee0 00000400:00000010:1.0:1773941570.238154:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833ee0. 00000100:00000001:1.0:1773941570.238158:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941570.238167:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10836680 x1860112291491072/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941586 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941570.238180:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10836680 x1860112291491072/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941586 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941570.238189:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941570.238191:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941570.238194:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941570.238197:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941570.238200:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941570.238203:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941570.238206:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941570.238214:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=71 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941570.238218:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941570.238222:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941570.238227:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:1.0:1773941570.238231:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941570.238234:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941570.238238:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941570.238243:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941570.238247:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941570.238251:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941570.238254:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941570.238256:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941570.238258:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941570.238263:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d10836680 x1860112291491072/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941586 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941570.238271:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941570.238273:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941570.238274:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941570.238283:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941570.238287:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941570.238289:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941570.238291:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941570.238293:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941570.238294:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941570.238296:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941570.238297:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941570.238320:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941570.238322:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941570.238324:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941570.238337:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 291 00000400:00000200:1.0:1773941570.238342:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941570.238345:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941570.238349:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941570.238352:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941570.238354:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941570.238356:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941570.238358:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941570.238360:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941570.238362:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941570.238364:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941570.238365:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941570.238367:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941570.238369:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941570.238371:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941570.238373:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941570.238374:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941570.238377:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941570.238380:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941570.238382:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941570.238384:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941570.238386:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941570.238391:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941570.238393:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941570.238395:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941570.238399:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c2300. 00000020:00000040:1.0:1773941570.238401:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941570.238404:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941570.238405:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941570.238407:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941570.238409:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941570.238411:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941570.238414:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d1118b000. 00000100:00000001:1.0:1773941570.238418:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941570.238422:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c2300 x1860112291491200/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941570.238429:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941570.238431:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941570.238435:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093c3ab5c] to pc [ptlrpcd_rcv+-1] req@ffff988d203c2300 x1860112291491200/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941570.238478:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941570.238481:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941570.238483:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941570.238486:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941570.238488:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941570.238490:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941570.238492:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6a80 going to sleep for 1 seconds 00000100:00000001:1.0:1773941570.238495:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941570.238497:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941570.238503:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10836680 x1860112291491072/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941586 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941570.238512:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941570.238516:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941570.238518:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941570.238521:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941570.238521:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941570.238528:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941570.238533:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941570.238540:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10836680 x1860112291491072/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941586 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941570.238546:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c2300 x1860112291491200/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941570.238585:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941570.238590:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941570.238592:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10836680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291491072:192.168.204.114@tcp:101:ll_cfg_requeue.0 02000000:00000001:2.0:1773941570.238595:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941570.238600:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941570.238608:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941570.238611:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941570.238615:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291491200:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941570.238621:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941570.238623:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941570.238628:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1773941570.238631:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 02000000:00000001:2.0:1773941570.238632:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941570.238639:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1773941570.238639:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000010:2.0:1773941570.238649:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a6400. 00000100:00000040:1.0:1773941570.238651:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10836680 x1860112291491072/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941586 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1773941570.238654:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941570.238669:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941570.238671:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1773941570.238675:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000100:00000010:1.0:1773941570.238677:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb6a80. 00000400:00000010:2.0:1773941570.238684:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000001:1.0:1773941570.238685:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941570.238689:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000200:2.0:1773941570.238693:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291491200, portal 25 00010000:00000001:1.0:1773941570.238695:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000001:2.0:1773941570.238701:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00000001:1.0:1773941570.238702:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000001:2.0:1773941570.238705:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631463680 : -113743078087936 : ffff988d203c2300) 00000020:00000001:1.0:1773941570.238707:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941570.238712:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00031e00 refcount=4 00000100:00000040:2.0:1773941570.238718:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c2300 x1860112291491200/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941580 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1773941570.238719:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090855936 : -113743618695680 : ffff988d00031e00) 00010000:00000001:1.0:1773941570.238743:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090855936 : -113743618695680 : ffff988d00031e00) 00010000:00010000:1.0:1773941570.238750:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2a1d lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941570.238751:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:1.0:1773941570.238758:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000040:2.0:1773941570.238764:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00010000:1.0:1773941570.238766:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2a1d lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941570.238768:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516110. 00000100:00000200:2.0:1773941570.238774:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291491200, offset 0 00010000:00000001:1.0:1773941570.238776:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941570.238780:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2a1d lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941570.238781:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941570.238790:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941570.238794:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:1.0:1773941570.238801:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941570.238805:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:2.0:1773941570.238809:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00010000:1.0:1773941570.238809:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2a1d lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941570.238816:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00000001:1.0:1773941570.238822:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:2.0:1773941570.238823:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941570.238825:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941570.238828:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941570.238831:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:2.0:1773941570.238836:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[114, 115] 00010000:00010000:1.0:1773941570.238837:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2a1d lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941570.238846:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00010000:00010000:1.0:1773941570.238851:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d00031e00 already unused, calling callback (ffffffffc12b4c20) 00000400:00000200:2.0:1773941570.238852:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [226:226:256:4294967295] 192.168.204.114@tcp LPNI seq info [116:116:8:4294967295] 10000000:00000001:1.0:1773941570.238855:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941570.238860:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2a1d lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941570.238875:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941570.238888:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000400:00000200:2.0:1773941570.238889:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00010000:00000001:1.0:1773941570.238892:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941570.238896:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941570.238898:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00031e00 refcount=5 00000800:00000200:2.0:1773941570.238903:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000020:00000001:1.0:1773941570.238904:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090855936 : -113743618695680 : ffff988d00031e00) 00010000:00000001:1.0:1773941570.238911:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090855936 : -113743618695680 : ffff988d00031e00) 00000800:00000010:2.0:1773941570.238914:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0dbd00. 00010000:00000001:1.0:1773941570.238914:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941570.238918:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2a1d lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941570.238926:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 10000000:00000001:1.0:1773941570.238928:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941570.238932:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2a1d lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1773941570.238940:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000800:00000200:2.0:1773941570.238941:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 10000000:00000040:1.0:1773941570.238943:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000800:00000200:2.0:1773941570.238948:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0dbd00 type 1, nob 616 niov 1 nkiov 1 10000000:00000001:1.0:1773941570.238952:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941570.238957:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941570.238959:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941570.238985:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941570.238997:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941570.239001:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941570.239003:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941570.239006:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d00031e00 with handle 0x8e86767622e2a1d from hash 00010000:00000001:1.0:1773941570.239009:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941570.239011:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941570.239012:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941570.239013:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941570.239014:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941570.239018:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000400:00000200:2.0:1773941570.239022:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941570.239023:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000100:00000001:2.0:1773941570.239027:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294961835 00010000:00000001:1.0:1773941570.239027:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941570.239031:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941570.239034:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941570.239036:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294961831 10000000:00000001:1.0:1773941570.239037:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941570.239044:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2a1d lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941570.239045:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294961835 00000100:00000001:2.0:1773941570.239052:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294961831 00010000:00000001:1.0:1773941570.239054:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941570.239056:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941570.239057:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941570.239060:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000100:00000001:2.0:1773941570.239061:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941570.239063:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941570.239066:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941570.239067:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941570.239068:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941570.239071:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941570.239072:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:1.0:1773941570.239074:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d00031e00/0x8e86767622e2a1d lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:0.0:1773941570.239077:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1773941570.239080:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941570.239086:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000800:00000010:0.0:1773941570.239087:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0dbd00. 00010000:00000040:1.0:1773941570.239088:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d21ec1400 count: 0 00000100:00000001:2.0:1773941570.239089:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941570.239092:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941570.239097:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941570.239100:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941570.239100:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000001:1.0:1773941570.239104:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941570.239108:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000200:0.0:1773941570.239112:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:1.0:1773941570.239114:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000400:00000200:0.0:1773941570.239118:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516110 00000400:00000010:0.0:1773941570.239123:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516110. 00000100:00000040:1.0:1773941570.239124:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10836680 x1860112291491072/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941586 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941570.239128:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941570.239136:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c2300 x1860112291491200/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941580 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941570.239148:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1773941570.239153:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1773941570.239154:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941570.239158:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d02ce7000. 00000100:00000040:0.0:1773941570.239159:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c2300 x1860112291491200/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941580 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1773941570.239164:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941570.239169:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:0.0:1773941570.239170:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941570.239172:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941570.239174:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:1.0:1773941570.239176:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941570.239196:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065c4a00. 02000000:00000001:1.0:1773941570.239200:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941570.239202:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941570.239203:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10836680. 00000100:00000001:1.0:1773941570.239207:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941570.239208:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941570.239210:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941570.239212:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941570.239214:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941570.239217:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d04d679c0. 00000020:00000010:1.0:1773941570.239221:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d181fc200. 00000020:00000010:1.0:1773941570.239225:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d181fd000. 00000040:00000010:1.0:1773941570.239228:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d181fce00. 00000020:00000010:1.0:1773941570.239232:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941570.239237:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d181fc800. 80000000:00000010:1.0:1773941570.239240:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d054f31e0. 00040000:00000010:1.0:1773941570.239244:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d181fd600. 20000000:00000010:1.0:1773941570.239248:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d181fd800. 10000000:00000040:1.0:1773941570.239251:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:1.0:1773941570.239254:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941570.239258:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d181fd000. 00000040:00000010:1.0:1773941570.239261:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d181fce00. 00000020:00000010:1.0:1773941570.239264:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941570.239267:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d181fc800. 80000000:00000010:1.0:1773941570.239269:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d054f31e0. 00040000:00000010:1.0:1773941570.239272:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d181fd600. 20000000:00000010:1.0:1773941570.239278:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d181fd800. 00000020:00000010:1.0:1773941570.239281:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d181fc200. 10000000:00000010:1.0:1773941570.239283:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d04d679c0. 10000000:01000000:1.0:1773941570.239285:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941570.239287:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941570.239289:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 00000800:00000001:0.0:1773941570.241541:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941570.241553:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941570.241556:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941570.241559:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941570.241565:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941570.241570:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941570.241578:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941570.241582:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941570.241586:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941570.241590:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324606d80 00000400:00000200:0.0:1773941570.241603:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941570.241627:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xf89 [1] + 0 00000800:00000001:0.0:1773941570.241635:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941570.241645:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941570.241648:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941570.241653:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941570.241658:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941570.241665:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c2300 x1860112291491200/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941580 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941570.241681:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c2300 x1860112291491200/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941580 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941570.241708:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941570.241718:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941570.241722:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941570.241821:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941570.241836:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941570.241842:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941570.241844:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941570.241847:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941570.241853:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941570.241856:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941570.241859:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941570.241894:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c2300 x1860112291491200/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941580 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941570.241908:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c2300 x1860112291491200/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941580 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941570.241934:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941570.241935:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941570.241937:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941570.241938:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941570.241940:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941570.241941:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941570.241943:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941570.241946:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=71 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941570.241948:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941570.241950:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941570.241951:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941570.241953:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941570.241955:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941570.241956:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941570.241960:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c2300 x1860112291491200/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941580 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941570.241964:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941570.241966:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941570.241967:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941570.241969:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941570.241973:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941570.241974:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941570.241975:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941570.241996:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941570.242003:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941570.242005:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941570.242008:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:2.0:1773941570.242011:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941570 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941570.242014:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941570.242020:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:2.0:1773941570.242022:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941570.242024:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941570.242025:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941570.242026:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941570.242029:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:2.0:1773941570.242032:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941570.242033:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941570.242036:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941570.242039:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:2.0:1773941570.242040:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941570.242263:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941570.242267:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941570.242275:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c2300 x1860112291491200/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941580 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941570.242284:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291491200:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941570.242286:0:14777:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:3.0:1773941570.242289:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941570.242292:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1773941570.242300:0:14777:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00000040:3.0:1773941570.242306:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c2300 x1860112291491200/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941580 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941570.242307:0:14777:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941570.242311:0:14777:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941570.242313:0:14777:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:2.0:1773941570.242317:0:14777:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:3.0:1773941570.242320:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:2.0:1773941570.242321:0:14777:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:3.0:1773941570.242324:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:2.0:1773941570.242326:0:14777:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941570.242327:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a6400. 00000001:00000001:2.0:1773941570.242328:0:14777:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:3.0:1773941570.242332:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:2.0:1773941570.242332:0:14777:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941570.242335:0:14777:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941570.242336:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941570.242339:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 10000000:00000001:2.0:1773941570.242340:0:14777:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941570.242343:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941570.242344:0:14777:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000010:3.0:1773941570.242346:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d1118b000. 00000100:00080000:2.0:1773941570.242349:0:14777:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 02000000:00000001:3.0:1773941570.242350:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941570.242353:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1773941570.242355:0:14777:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000010:3.0:1773941570.242356:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c2300. 00000100:00000001:2.0:1773941570.242358:0:14777:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941570.242362:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941570.242362:0:14777:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941570.242365:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1773941570.242365:0:14777:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:3.0:1773941570.242368:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941570.242371:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941570.242372:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941570.242378:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941570.242378:0:14777:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:3.0:1773941570.242383:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:01000000:2.0:1773941570.242383:0:14777:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:2.0:1773941570.242388:0:14777:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:2.0:1773941570.242394:0:14777:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:2.0:1773941570.242395:0:14777:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941570.242397:0:14777:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:2.0:1773941570.242400:0:14777:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:2.0:1773941570.242404:0:14777:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941570.242406:0:14777:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941570.242408:0:14777:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941570.242410:0:14777:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941570.242412:0:14777:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941570.246083:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941570.246086:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941570.246088:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941570.246092:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00010000:00000010:1.1:1773941570.247993:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d21ec1400. 00010000:00000010:1.1:1773941570.248000:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d00031e00. 00000020:00000001:2.0:1773941570.555124:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941570.555132:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941570.555137:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1773941571.515088:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294961835, at 4294961856 00000100:00000010:0.0:1773941571.515109:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580b40. 00000100:00000040:1.0:1773941571.515127:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294961835, at 4294961856 00000100:00000010:1.0:1773941571.515145:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580000. 00000020:00000001:2.0:1773941571.579175:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941571.579185:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941571.579194:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1773941571.835191:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) c1c1ff21-5f68-4bf6-bdc4-258644b24465->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1773941571.835204:0:8:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:0.0:1773941571.835220:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10835880. 00000020:00000040:0.0:1773941571.835228:0:8:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941571.835233:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941571.835236:0:8:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941571.835244:0:8:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941571.835255:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff988d0a20bd00. 00000100:00000001:0.0:1773941571.835263:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941571.835277:0:8:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging c1c1ff21-5f68-4bf6-bdc4-258644b24465->MGS req@ffff988d10835880 x1860112291491328/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1773941571.835287:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941571.835292:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941571.835299:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c1418601] to pc [ptlrpcd_00_01+1] req@ffff988d10835880 x1860112291491328/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941571.835368:0:8:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941571.835374:0:8:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (299) 00000100:00000001:0.0:1773941571.835393:0:2401:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1773941571.835398:0:2401:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:1.0:1773941571.835401:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941571.835401:0:2401:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941571.835412:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941571.835414:0:2401:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941571.835418:0:2401:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941571.835423:0:2401:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941571.835426:0:2401:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941571.835428:0:2401:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941571.835436:0:2401:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10835880 x1860112291491328/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941571.835443:0:2401:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941571.835446:0:2401:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941571.835450:0:2401:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941571.835461:0:2401:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941571.835465:0:2401:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941571.835468:0:2401:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10835880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2401:1860112291491328:192.168.204.114@tcp:400:kworker.0 00000100:00000001:0.0:1773941571.835472:0:2401:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941571.835479:0:2401:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941571.835481:0:2401:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941571.835484:0:2401:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941571.835489:0:2401:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff988d065c4200. 02000000:00000001:0.0:1773941571.835491:0:2401:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941571.835499:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941571.835521:0:2401:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0014bd10. 00000400:00000010:0.0:1773941571.835529:0:2401:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833ee0. 00000100:00000200:0.0:1773941571.835538:0:2401:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112291491328, portal 25 00000100:00000001:0.0:1773941571.835540:0:2401:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941571.835541:0:2401:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367694976 : -113743341856640 : ffff988d10835880) 00000100:00000040:0.0:1773941571.835546:0:2401:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10835880 x1860112291491328/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941587 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941571.835563:0:2401:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941571.835565:0:2401:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941571.835568:0:2401:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d108335d8. 00000100:00000200:0.0:1773941571.835571:0:2401:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1860112291491328, offset 0 00000400:00000200:0.0:1773941571.835577:0:2401:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941571.835586:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941571.835611:0:2401:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941571.835628:0:2401:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941571.835633:0:2401:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941571.835637:0:2401:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941571.835639:0:2401:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941571.835642:0:2401:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[116, 115] 00000400:00000200:0.0:1773941571.835645:0:2401:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:0.0:1773941571.835649:0:2401:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [227:227:256:4294967295] 192.168.204.124@tcp LPNI seq info [117:117:8:4294967295] 00000400:00000200:0.0:1773941571.835653:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941571.835664:0:2401:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:0.0:1773941571.835670:0:2401:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:0.0:1773941571.835678:0:2401:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d113e9b00. 00000800:00000200:0.0:1773941571.835686:0:2401:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:0.0:1773941571.835694:0:2401:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:0.0:1773941571.835728:0:2401:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d113e9b00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:0.0:1773941571.835746:0:2401:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941571.835750:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941571.835753:0:2401:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294963431 00000100:00000010:0.0:1773941571.835757:0:2401:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580b40. 00000100:00000001:0.0:1773941571.835765:0:2401:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1773941571.835772:0:2401:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294963431 00000100:00000010:0.0:1773941571.835774:0:2401:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580300. 00000100:00000001:0.0:1773941571.835777:0:2401:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1773941571.835785:0:2401:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941571.835787:0:2401:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941571.835790:0:2401:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941571.835793:0:2401:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941571.836025:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941571.836032:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d113e9b00. 00000400:00000200:1.0:1773941571.836060:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941571.836071:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941571.836080:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d108335d8 00000400:00000010:1.0:1773941571.836083:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d108335d8. 00000100:00000001:1.0:1773941571.836089:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941571.836099:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10835880 x1860112291491328/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941587 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941571.836113:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941571.836118:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10835880 x1860112291491328/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941587 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941571.836126:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941571.836129:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941571.839028:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941571.839042:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941571.839045:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941571.839047:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941571.839054:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941571.839059:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941571.839065:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941571.839070:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941571.839075:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941571.839079:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324606e00 00000400:00000200:0.0:1773941571.839083:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941571.839087:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xf91 [1] + 224 00000800:00000001:0.0:1773941571.839095:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941571.839103:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941571.839105:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941571.839109:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941571.839115:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941571.839121:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10835880 x1860112291491328/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941587 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941571.839132:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff988d10835880 x1860112291491328/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941587 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941571.839151:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941571.839158:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941571.839160:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941571.839191:0:2401:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941571.839195:0:2401:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941571.839201:0:2401:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941571.839203:0:2401:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941571.839206:0:2401:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0014bd10. 00000400:00000200:1.0:1773941571.839210:0:2401:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833ee0 00000400:00000010:1.0:1773941571.839238:0:2401:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833ee0. 00000100:00000001:1.0:1773941571.839241:0:2401:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941571.839248:0:2401:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10835880 x1860112291491328/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941587 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941571.839257:0:2401:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10835880 x1860112291491328/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941587 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941571.839264:0:2401:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941571.839265:0:2401:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941571.839267:0:2401:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941571.839270:0:2401:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941571.839273:0:2401:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941571.839275:0:2401:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941571.839278:0:2401:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941571.839284:0:2401:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=72 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941571.839287:0:2401:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941571.839290:0:2401:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:1.0:1773941571.839292:0:2401:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941571.839295:0:2401:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941571.839297:0:2401:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941571.839313:0:2401:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941571.839317:0:2401:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941571.839320:0:2401:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941571.839323:0:2401:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941571.839324:0:2401:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941571.839326:0:2401:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941571.839329:0:2401:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d10835880 x1860112291491328/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941587 ref 1 fl Rpc:RNQU/200/0 rc 0/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941571.839335:0:2401:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941571.839336:0:2401:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941571.839338:0:2401:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941571.839341:0:2401:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941571.839343:0:2401:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941571.839349:0:2401:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941571.839351:0:2401:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941571.839352:0:2401:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941571.839353:0:2401:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941571.839354:0:2401:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941571.839355:0:2401:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941571.839357:0:2401:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941571.839359:0:2401:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941571.839362:0:2401:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941571.839364:0:2401:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 293 00000400:00000200:1.0:1773941571.839371:0:2401:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941571.839375:0:2401:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941571.839378:0:2401:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941571.839381:0:2401:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941571.839383:0:2401:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941571.839385:0:2401:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941571.839388:0:2401:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941571.839390:0:2401:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941571.839393:0:2401:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941571.839395:0:2401:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941571.839396:0:2401:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941571.839398:0:2401:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941571.839401:0:2401:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941571.839402:0:2401:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941571.839403:0:2401:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941571.839405:0:2401:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941571.839408:0:2401:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941571.839410:0:2401:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941571.839413:0:2401:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941571.839415:0:2401:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941571.839418:0:2401:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941571.839420:0:2401:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941571.839428:0:2401:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941571.839429:0:2401:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941571.839434:0:2401:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c1180. 00000020:00000040:1.0:1773941571.839437:0:2401:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941571.839440:0:2401:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941571.839441:0:2401:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941571.839444:0:2401:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941571.839446:0:2401:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941571.839451:0:2401:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941571.839456:0:2401:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d1118bc00. 00000100:00000001:1.0:1773941571.839461:0:2401:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941571.839466:0:2401:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c1180 x1860112291491456/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941571.839474:0:2401:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941571.839477:0:2401:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941571.839481:0:2401:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000223b0731] to pc [ptlrpcd_rcv+-1] req@ffff988d203c1180 x1860112291491456/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941571.839501:0:2401:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941571.839503:0:2401:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941571.839504:0:2401:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:1.0:1773941571.839509:0:2401:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10835880 x1860112291491328/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941587 ref 1 fl Rpc:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941571.839516:0:2401:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941571.839519:0:2401:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941571.839522:0:2401:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941571.839528:0:2401:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10835880 x1860112291491328/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941587 ref 1 fl Interpret:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941571.839535:0:2401:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10835880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2401:1860112291491328:192.168.204.114@tcp:400:kworker.0 00000100:00000001:3.0:1773941571.839538:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941571.839541:0:2401:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941571.839544:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941571.839545:0:2401:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1773941571.839547:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941571.839563:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c1180 x1860112291491456/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941571.839564:0:2401:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10835880 x1860112291491328/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941587 ref 1 fl Complete:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941571.839635:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941571.839636:0:2401:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1773941571.839638:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941571.839643:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941571.839644:0:2401:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1773941571.839647:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000010:1.0:1773941571.839648:0:2401:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff988d065c4200. 02000000:00000001:3.0:1773941571.839657:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941571.839659:0:2401:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941571.839664:0:2401:0:(genops.c:1236:class_import_put()) Process entered 00000100:00100000:3.0:1773941571.839667:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291491456:192.168.204.114@tcp:250:ptlrpcd_00_00.0 00000020:00000040:1.0:1773941571.839668:0:2401:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941571.839682:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000001:1.0:1773941571.839682:0:2401:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000001:3.0:1773941571.839695:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000010:1.0:1773941571.839695:0:2401:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff988d0a20bd00. 02000000:00000001:3.0:1773941571.839700:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941571.839706:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1773941571.839707:0:2401:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941571.839711:0:2401:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941571.839714:0:2401:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10835880. 00000100:00000001:1.0:1773941571.839717:0:2401:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941571.839719:0:2401:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941571.839722:0:2401:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000010:3.0:1773941571.839725:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03522c00. 02000000:00000001:3.0:1773941571.839728:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941571.839729:0:2401:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:3.0:1773941571.839734:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941571.839734:0:2401:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1773941571.839749:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00000100:00000001:1.0:1773941571.839749:0:2401:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941571.839755:0:2401:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1773941571.839760:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00000100:00000200:3.0:1773941571.839765:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291491456, portal 25 00000100:00000001:3.0:1773941571.839767:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941571.839770:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631459200 : -113743078092416 : ffff988d203c1180) 00000100:00000040:3.0:1773941571.839777:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c1180 x1860112291491456/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941581 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941571.839784:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941571.839787:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:3.0:1773941571.839789:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac220. 00000100:00000200:3.0:1773941571.839792:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291491456, offset 0 00000400:00000200:3.0:1773941571.839796:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941571.839800:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941571.839804:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941571.839807:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941571.839810:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941571.839812:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941571.839814:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941571.839817:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[116, 117] 00000400:00000200:3.0:1773941571.839820:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941571.839823:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [228:228:256:4294967295] 192.168.204.114@tcp LPNI seq info [118:118:8:4294967295] 00000400:00000200:3.0:1773941571.839826:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941571.839849:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:3.0:1773941571.839853:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:3.0:1773941571.839857:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74c00. 00000800:00000200:3.0:1773941571.839862:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:3.0:1773941571.839869:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:3.0:1773941571.839872:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74c00 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:3.0:1773941571.839889:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941571.839892:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:3.0:1773941571.839895:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294963435 00000100:00000001:3.0:1773941571.839900:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294963431 00000100:00000001:3.0:1773941571.839905:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294963435 00000100:00000001:3.0:1773941571.839910:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294963431 00000100:00000001:3.0:1773941571.839918:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941571.839921:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941571.839924:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941571.839927:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941571.839933:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941571.839935:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1773941571.839937:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941571.839938:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941571.839940:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941571.839942:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941571.840020:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941571.840023:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74c00. 00000400:00000200:1.0:1773941571.840027:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941571.840032:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941571.840036:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac220 00000400:00000010:1.0:1773941571.840038:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac220. 00000100:00000001:1.0:1773941571.840040:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941571.840044:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c1180 x1860112291491456/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941581 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941571.840050:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941571.840053:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c1180 x1860112291491456/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941581 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941571.840058:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941571.840059:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941571.841646:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941571.841656:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941571.841659:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941571.841662:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941571.841677:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941571.841692:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941571.841698:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941571.841701:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941571.841704:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941571.841706:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324606e80 00000400:00000200:0.0:1773941571.841710:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941571.841712:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xfa1 [1] + 0 00000800:00000001:0.0:1773941571.841717:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941571.841722:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941571.841724:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941571.841727:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941571.841730:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941571.841735:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c1180 x1860112291491456/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941581 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941571.841744:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c1180 x1860112291491456/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941581 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941571.841760:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941571.841764:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941571.841766:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941571.841785:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941571.841788:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941571.841792:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941571.841794:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941571.841796:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941571.841801:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941571.841803:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941571.841805:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941571.841810:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c1180 x1860112291491456/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941581 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941571.841819:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c1180 x1860112291491456/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941581 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941571.841825:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941571.841826:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941571.841828:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941571.841830:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941571.841848:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941571.841850:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941571.841865:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941571.841870:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=72 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941571.841873:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941571.841876:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941571.841877:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941571.841880:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941571.841882:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941571.841884:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941571.841888:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c1180 x1860112291491456/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941581 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941571.841894:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941571.841896:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941571.841897:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941571.841899:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941571.841906:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941571.841908:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941571.841909:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941571.841912:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941571.841914:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941571.841916:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941571.841918:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941571.841922:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941571 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941571.841924:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941571.841931:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941571.841933:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941571.841935:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941571.841937:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941571.841938:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941571.841941:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941571.841943:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941571.841946:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941571.841949:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941571.841952:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941571.841958:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941571.842204:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941571.842210:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941571.842220:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c1180 x1860112291491456/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941581 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941571.842222:0:14779:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00080000:3.0:1773941571.842238:0:14779:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:2.0:1773941571.842239:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291491456:192.168.204.114@tcp:250:ptlrpcd_00_00.0 00000100:00100000:3.0:1773941571.842245:0:14779:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941571.842249:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941571.842250:0:14779:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941571.842253:0:14779:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000001:2.0:1773941571.842255:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000001:00000001:3.0:1773941571.842258:0:14779:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000040:2.0:1773941571.842263:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c1180 x1860112291491456/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941581 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_00.0' uid:0 gid:0 projid:4294967295 00000001:00000001:3.0:1773941571.842264:0:14779:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941571.842271:0:14779:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941571.842285:0:14779:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:2.0:1773941571.842286:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:3.0:1773941571.842296:0:14779:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941571.842296:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:3.0:1773941571.842307:0:14779:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941571.842308:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03522c00. 10000000:00000001:3.0:1773941571.842319:0:14779:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941571.842322:0:14779:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000001:2.0:1773941571.842323:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941571.842330:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000100:00080000:3.0:1773941571.842335:0:14779:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000020:00000040:2.0:1773941571.842336:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941571.842342:0:14779:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:3.0:1773941571.842344:0:14779:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941571.842346:0:14779:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1773941571.842356:0:14779:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000020:00000001:2.0:1773941571.842357:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941571.842366:0:14779:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000010:2.0:1773941571.842366:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d1118bc00. 10000000:01000000:3.0:1773941571.842368:0:14779:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:3.0:1773941571.842374:0:14779:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 02000000:00000001:2.0:1773941571.842375:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941571.842378:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941571.842380:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c1180. 00000100:00000001:2.0:1773941571.842385:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941571.842386:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941571.842388:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941571.842393:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1773941571.842396:0:14779:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:3.0:1773941571.842399:0:14779:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941571.842404:0:14779:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941571.842404:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941571.842415:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:02000000:3.0:1773941571.842420:0:14779:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:2.0:1773941571.842421:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941571.842427:0:14779:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941571.842431:0:14779:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941571.842433:0:14779:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941571.842436:0:14779:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941571.842438:0:14779:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941571.939120:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941571.939152:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941571.939158:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941571.939162:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941571.939165:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941571.939166:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941571.939171:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941571.939173:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941571.939193:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:1.0:1773941571.939196:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941571.939198:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941571.939212:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c0700. 00000020:00000040:1.0:1773941571.939219:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941571.939226:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941571.939230:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941571.939237:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941571.939262:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d181fd600. 00000100:00000001:1.0:1773941571.939271:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941571.939277:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941571.939281:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941571.939290:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d04dd4e00. 00010000:00000001:1.0:1773941571.939298:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941571.939302:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d01488800. 00000020:00000001:1.0:1773941571.939308:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941571.939312:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d01488800 with handle 0x8e86767622e2a24 to hash 00000020:00000001:1.0:1773941571.939315:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941571.939317:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941571.939321:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941571.939325:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941571.939329:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941571.939335:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a24 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941571.939347:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a24 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941571.939375:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a24 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941571.939383:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941571.939386:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941571.939394:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941571.939397:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330528610752 : -113743180940864 : ffff988d1a1ab9c0) 00000100:00000001:1.0:1773941571.939401:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941571.939403:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00000100:00000001:1.0:1773941571.939409:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941571.939415:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941571.939418:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941571.939420:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941571.939434:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c0700 x1860112291491584/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941571.939444:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941571.939447:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941571.939452:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941571.939455:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941571.939459:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941571.939463:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291491584:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941571.939468:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941571.939475:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941571.939478:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941571.939482:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941571.939488:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d11189400. 02000000:00000001:1.0:1773941571.939490:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941571.939500:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941571.939506:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c5d8. 00000400:00000010:1.0:1773941571.939514:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000100:00000200:1.0:1773941571.939523:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291491584, portal 25 00000100:00000001:1.0:1773941571.939525:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941571.939533:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00000100:00000040:1.0:1773941571.939540:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c0700 x1860112291491584/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941587 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941571.939549:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941571.939553:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941571.939556:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113dedd0. 00000100:00000200:1.0:1773941571.939560:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291491584, offset 0 00000400:00000200:1.0:1773941571.939569:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941571.939580:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941571.939590:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941571.939597:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941571.939602:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941571.939608:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941571.939611:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941571.939615:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[118, 117] 00000400:00000200:1.0:1773941571.939620:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:1.0:1773941571.939625:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [229:229:256:4294967295] 192.168.204.124@tcp LPNI seq info [119:119:8:4294967295] 00000400:00000200:1.0:1773941571.939630:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941571.939641:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:1.0:1773941571.939648:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:1.0:1773941571.939654:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfd700. 00000800:00000200:1.0:1773941571.939672:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:1.0:1773941571.939680:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:1.0:1773941571.939685:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfd700 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941571.939702:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941571.939705:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941571.939708:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294963535 00000100:00000001:1.0:1773941571.939717:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294963435 00000100:00000001:1.0:1773941571.939722:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294963535 00000100:00000001:1.0:1773941571.939725:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294963435 00000100:00000001:1.0:1773941571.939732:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941571.939734:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941571.939736:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941571.939738:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941571.939741:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 11 seconds 00000100:00000001:1.0:1773941571.939743:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941571.939746:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941571.939939:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941571.939946:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfd700. 00000400:00000200:0.0:1773941571.939952:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941571.939958:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941571.939988:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113dedd0 00000400:00000010:0.0:1773941571.939990:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113dedd0. 00000100:00000001:0.0:1773941571.939994:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941571.939999:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c0700 x1860112291491584/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941587 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941571.940008:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941571.940011:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d203c0700 x1860112291491584/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941587 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941571.940031:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941571.940033:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941571.940974:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941571.940983:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941571.940986:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941571.940988:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941571.940994:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941571.941000:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941571.941007:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941571.941012:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941571.941017:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941571.941022:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324606f00 00000400:00000200:0.0:1773941571.941027:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941571.941034:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xfa9 [1] + 224 00000800:00000001:0.0:1773941571.941043:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941571.941052:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941571.941055:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941571.941059:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941571.941064:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941571.941067:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c0700 x1860112291491584/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941587 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941571.941075:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d203c0700 x1860112291491584/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941587 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941571.941089:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941571.941095:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941571.941098:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941571.941123:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941571.941130:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941571.941131:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941571.941134:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:1.0:1773941571.941137:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:1.0:1773941571.941139:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:1.0:1773941571.941141:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941571.941146:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c0700 x1860112291491584/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941587 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941571.941155:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c0700 x1860112291491584/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941587 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941571.941161:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941571.941162:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941571.941165:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941571.941167:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941571.941169:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941571.941170:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941571.941172:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941571.941177:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=72 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941571.941180:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=0 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941571.941183:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941571.941186:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:1.0:1773941571.941189:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941571.941191:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941571.941194:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941571.941197:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941571.941200:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941571.941210:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941571.941212:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941571.941213:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941571.941215:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941571.941218:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d203c0700 x1860112291491584/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941587 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941571.941224:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941571.941226:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941571.941227:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941571.941230:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941571.941232:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941571.941233:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941571.941235:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941571.941236:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941571.941237:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941571.941238:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941571.941239:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941571.941242:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941571.941243:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941571.941246:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941571.941248:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 294 00000400:00000200:1.0:1773941571.941252:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941571.941256:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941571.941259:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941571.941261:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941571.941263:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941571.941265:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941571.941267:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941571.941269:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941571.941271:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941571.941273:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941571.941275:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941571.941276:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941571.941284:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941571.941285:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941571.941287:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941571.941288:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941571.941291:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941571.941293:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941571.941295:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941571.941298:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941571.941302:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941571.941305:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941571.941307:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941571.941310:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941571.941314:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c1c00. 00000020:00000040:1.0:1773941571.941317:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941571.941321:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941571.941323:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941571.941326:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941571.941328:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941571.941331:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941571.941336:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d11188800. 00000100:00000001:1.0:1773941571.941340:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941571.941347:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c1c00 x1860112291491712/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941571.941355:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941571.941357:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941571.941361:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000033e82992] to pc [ptlrpcd_rcv+-1] req@ffff988d203c1c00 x1860112291491712/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941571.941379:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941571.941381:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941571.941383:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941571.941385:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941571.941387:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941571.941393:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941571.941394:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941571.941396:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941571.941398:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941571.941402:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c0700 x1860112291491584/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941587 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941571.941407:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941571.941409:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941571.941411:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941571.941413:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941571.941416:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c0700 x1860112291491584/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941587 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941571.941421:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291491584:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941571.941424:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941571.941425:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941571.941427:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941571.941428:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941571.941430:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c0700 x1860112291491584/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941587 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941571.941435:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941571.941437:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941571.941439:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941571.941440:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:1.0:1773941571.941443:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941571.941445:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941571.941446:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941571.941447:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488800 refcount=4 00000100:00000001:2.0:1773941571.941449:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1773941571.941449:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941571.941454:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00000100:00000001:2.0:1773941571.941455:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941571.941458:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00010000:00010000:1.0:1773941571.941458:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a24 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941571.941467:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c1c00 x1860112291491712/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941571.941473:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941571.941480:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00010000:1.0:1773941571.941480:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a24 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941571.941483:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941571.941488:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00010000:00000001:1.0:1773941571.941489:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 02000000:00000001:2.0:1773941571.941491:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00010000:00010000:1.0:1773941571.941493:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a24 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941571.941494:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941571.941501:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291491712:192.168.204.114@tcp:250:ll_cfg_requeue.0 00010000:00000001:1.0:1773941571.941503:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941571.941505:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00000001:1.0:1773941571.941506:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 02000000:00000001:2.0:1773941571.941508:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00010000:1.0:1773941571.941510:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a24 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941571.941511:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941571.941516:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:1.0:1773941571.941520:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941571.941521:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941571.941522:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a7800. 00010000:00000001:1.0:1773941571.941524:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 02000000:00000001:2.0:1773941571.941525:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941571.941528:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a24 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941571.941532:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941571.941537:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00010000:00010000:1.0:1773941571.941538:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d01488800 already unused, calling callback (ffffffffc12b4c20) 00000400:00000010:2.0:1773941571.941542:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 10000000:00000001:1.0:1773941571.941542:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941571.941544:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a24 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:2.0:1773941571.941547:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291491712, portal 25 00000100:00000001:2.0:1773941571.941552:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00000001:1.0:1773941571.941553:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000100:00000001:2.0:1773941571.941554:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631461888 : -113743078089728 : ffff988d203c1c00) 00010000:00000001:1.0:1773941571.941556:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941571.941557:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941571.941558:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488800 refcount=5 00000020:00000001:1.0:1773941571.941559:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941571.941561:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941571.941565:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000100:00000040:2.0:1773941571.941569:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c1c00 x1860112291491712/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941581 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:1.0:1773941571.941574:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a24 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941571.941578:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941571.941584:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 10000000:00000001:1.0:1773941571.941588:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000010:2.0:1773941571.941589:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516e58. 10000000:00010000:1.0:1773941571.941591:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a24 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:2.0:1773941571.941594:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291491712, offset 0 10000000:01000000:1.0:1773941571.941599:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:2.0:1773941571.941604:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 10000000:00000040:1.0:1773941571.941604:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941571.941608:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941571.941611:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000400:00000200:2.0:1773941571.941612:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941571.941614:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941571.941618:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000400:00000200:2.0:1773941571.941619:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:1.0:1773941571.941620:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000400:00000200:2.0:1773941571.941624:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:1.0:1773941571.941624:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941571.941626:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941571.941630:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000020:00000040:1.0:1773941571.941630:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d01488800 with handle 0x8e86767622e2a24 from hash 00000400:00000200:2.0:1773941571.941634:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941571.941635:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000400:00000200:2.0:1773941571.941638:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941571.941638:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941571.941640:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941571.941643:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000400:00000200:2.0:1773941571.941644:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[118, 119] 00010000:00000001:1.0:1773941571.941646:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941571.941650:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000400:00000200:2.0:1773941571.941651:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00010000:00000001:1.0:1773941571.941654:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941571.941656:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:2.0:1773941571.941657:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [230:230:256:4294967295] 192.168.204.114@tcp LPNI seq info [120:120:8:4294967295] 00010000:00000001:1.0:1773941571.941658:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941571.941662:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941571.941666:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:1.0:1773941571.941667:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a24 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941571.941677:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:2.0:1773941571.941680:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00010000:00000001:1.0:1773941571.941680:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941571.941683:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941571.941687:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941571.941690:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000200:2.0:1773941571.941691:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00010000:00000001:1.0:1773941571.941694:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000010:2.0:1773941571.941706:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0dbb00. 00010000:00000001:1.0:1773941571.941710:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000200:2.0:1773941571.941713:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00010000:00010000:1.0:1773941571.941713:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a24 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:1.0:1773941571.941722:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d04dd4e00 count: 0 00000800:00000200:2.0:1773941571.941723:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:2.0:1773941571.941728:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0dbb00 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941571.941734:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941571.941736:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941571.941738:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941571.941740:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941571.941743:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c0700 x1860112291491584/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941587 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1773941571.941747:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941571.941752:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294963537 00000100:00000001:1.0:1773941571.941755:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1773941571.941758:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294963535 02000000:00000001:1.0:1773941571.941759:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941571.941779:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d11189400. 00000100:00000001:2.0:1773941571.941782:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294963537 02000000:00000001:1.0:1773941571.941785:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1773941571.941788:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294963535 00000020:00000001:1.0:1773941571.941790:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941571.941794:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941571.941798:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941571.941801:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d181fd600. 00000100:00000001:2.0:1773941571.941809:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941571.941810:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941571.941814:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941571.941818:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c0700. 00000100:00000001:1.0:1773941571.941823:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941571.941824:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941571.941825:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:0.0:1773941571.941852:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 10000000:00000001:1.0:1773941571.941858:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941571.941859:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1773941571.941862:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0dbb00. 10000000:01000000:1.0:1773941571.941868:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000100:00000001:2.0:1773941571.941869:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941571.941901:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 10000000:00000001:1.0:1773941571.941903:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000100:00000001:2.0:1773941571.941907:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 10000000:00000010:1.0:1773941571.941910:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d04d679c0. 00000100:00000001:2.0:1773941571.941912:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000400:00000200:0.0:1773941571.941916:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:2.0:1773941571.941919:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000010:1.0:1773941571.941920:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d181fd600. 00000100:00000001:2.0:1773941571.941924:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000200:0.0:1773941571.941925:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516e58 00000100:00000001:2.0:1773941571.941928:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941571.941929:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d181fc800. 00000100:00000001:2.0:1773941571.941933:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000040:00000010:1.0:1773941571.941938:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d181fce00. 00000400:00000010:0.0:1773941571.941942:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516e58. 00000020:00000010:1.0:1773941571.941946:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d18b18000. 00000100:00000001:0.0:1773941571.941949:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000020:00000010:1.0:1773941571.941953:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d181fd000. 00000100:00000200:0.0:1773941571.941958:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c1c00 x1860112291491712/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941581 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 80000000:00000010:1.0:1773941571.941961:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d054f31e0. 00040000:00000010:1.0:1773941571.941970:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d181fc400. 00000100:00000001:0.0:1773941571.941973:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941571.941995:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c1c00 x1860112291491712/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941581 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 20000000:00000010:1.0:1773941571.942027:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d181fd800. 00000100:00000001:0.0:1773941571.942033:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941571.942040:0:2393:0:(events.c:68:request_out_callback()) Process leaving 10000000:00000040:1.0:1773941571.942044:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:1.0:1773941571.942048:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941571.942054:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d181fc800. 00000040:00000010:1.0:1773941571.942059:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d181fce00. 00000020:00000010:1.0:1773941571.942063:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941571.942069:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d181fd000. 80000000:00000010:1.0:1773941571.942074:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d054f31e0. 00040000:00000010:1.0:1773941571.942079:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d181fc400. 20000000:00000010:1.0:1773941571.942083:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d181fd800. 00000020:00000010:1.0:1773941571.942087:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d181fd600. 10000000:00000010:1.0:1773941571.942091:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d04d679c0. 10000000:01000000:1.0:1773941571.942110:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941571.942114:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941571.942117:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941571.942119:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941571.942121:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941571.942123:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941571.942126:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941571.943487:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941571.943494:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941571.943497:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941571.943499:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941571.943503:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941571.943506:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941571.943511:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941571.943513:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941571.943516:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941571.943518:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324606f80 00000400:00000200:0.0:1773941571.943521:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941571.943523:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xfb9 [1] + 0 00000800:00000001:0.0:1773941571.943527:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941571.943532:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941571.943534:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941571.943536:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941571.943539:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941571.943543:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c1c00 x1860112291491712/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941581 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941571.943551:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c1c00 x1860112291491712/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941581 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941571.943564:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941571.943576:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941571.943579:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941571.943619:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941571.943622:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941571.943625:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941571.943627:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941571.943628:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941571.943632:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941571.943633:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941571.943635:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941571.943640:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c1c00 x1860112291491712/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941581 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941571.943648:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c1c00 x1860112291491712/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941581 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941571.943653:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941571.943654:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941571.943656:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941571.943657:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941571.943658:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941571.943659:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941571.943661:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941571.943663:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=72 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941571.943665:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=0 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941571.943667:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941571.943668:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941571.943670:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941571.943672:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941571.943674:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941571.943684:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c1c00 x1860112291491712/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941581 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941571.943689:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941571.943691:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941571.943692:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941571.943694:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941571.943697:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941571.943698:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941571.943699:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941571.943701:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941571.943702:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941571.943704:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941571.943706:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:2.0:1773941571.943709:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941571 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941571.943711:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941571.943717:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:2.0:1773941571.943719:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941571.943720:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941571.943721:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941571.943723:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941571.943725:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:2.0:1773941571.943727:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941571.943729:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941571.943732:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941571.943734:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:2.0:1773941571.943736:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941571.944019:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941571.944025:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941571.944034:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c1c00 x1860112291491712/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941581 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941571.944040:0:14780:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00080000:2.0:1773941571.944053:0:14780:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:2.0:1773941571.944060:0:14780:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00100000:3.0:1773941571.944071:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291491712:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941571.944078:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941571.944081:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941571.944085:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c1c00 x1860112291491712/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941581 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941571.944089:0:14780:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941571.944093:0:14780:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000001:3.0:1773941571.944111:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941571.944114:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:2.0:1773941571.944114:0:14780:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000010:3.0:1773941571.944117:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a7800. 02000000:00000001:3.0:1773941571.944120:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941571.944121:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941571.944125:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941571.944127:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:2.0:1773941571.944127:0:14780:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000010:3.0:1773941571.944132:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d11188800. 00000001:00000001:2.0:1773941571.944133:0:14780:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941571.944135:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941571.944139:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000001:00000001:2.0:1773941571.944140:0:14780:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000010:3.0:1773941571.944142:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c1c00. 00000001:00000001:2.0:1773941571.944143:0:14780:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941571.944147:0:14780:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941571.944148:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 10000000:00000001:2.0:1773941571.944150:0:14780:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941571.944152:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941571.944153:0:14780:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941571.944155:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1773941571.944159:0:14780:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:3.0:1773941571.944161:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941571.944163:0:14780:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:3.0:1773941571.944165:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941571.944167:0:14780:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941571.944169:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941571.944171:0:14780:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941571.944172:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1773941571.944176:0:14780:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941571.944196:0:14780:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941571.944198:0:14780:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:2.0:1773941571.944200:0:14780:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:2.0:1773941571.944216:0:14780:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:2.0:1773941571.944219:0:14780:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941571.944222:0:14780:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:2.0:1773941571.944228:0:14780:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:2.0:1773941571.944232:0:14780:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941571.944235:0:14780:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941571.944238:0:14780:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941571.944240:0:14780:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941571.944257:0:14780:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941571.947014:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d04dd4e00. 00010000:00000010:1.1:1773941571.947030:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d01488800. 00000020:00000001:2.0:1773941572.603183:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941572.603191:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941572.603196:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941573.243088:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294963537, at 4294963584 00000100:00000040:0.0:1773941573.243093:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294963537, at 4294963584 00000100:00000010:1.0:1773941573.243111:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580300. 00000100:00000010:0.0:1773941573.243116:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580b40. 10000000:00000001:1.0:1773941573.499086:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941573.499095:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941573.499100:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941573.499102:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941573.499103:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941573.499105:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941573.499108:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941573.499109:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941573.499112:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:1.0:1773941573.499115:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941573.499116:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941573.499129:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c0700. 00000020:00000040:1.0:1773941573.499134:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941573.499139:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941573.499161:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941573.499168:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941573.499176:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d181fd800. 00000100:00000001:1.0:1773941573.499184:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941573.499189:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941573.499193:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941573.499201:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d04dd4e00. 00010000:00000001:1.0:1773941573.499207:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941573.499210:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d01488800. 00000020:00000001:1.0:1773941573.499214:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941573.499216:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d01488800 with handle 0x8e86767622e2a2b to hash 00000020:00000001:1.0:1773941573.499218:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941573.499219:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941573.499222:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941573.499227:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941573.499229:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941573.499232:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a2b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941573.499241:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a2b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941573.499252:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a2b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941573.499257:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941573.499258:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941573.499265:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941573.499266:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330528610752 : -113743180940864 : ffff988d1a1ab9c0) 00000100:00000001:1.0:1773941573.499268:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941573.499269:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00000100:00000001:1.0:1773941573.499274:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941573.499279:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941573.499281:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941573.499287:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941573.499299:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c0700 x1860112291491840/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941573.499306:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941573.499309:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941573.499318:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941573.499320:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941573.499322:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941573.499325:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291491840:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941573.499329:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941573.499334:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941573.499336:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941573.499338:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941573.499342:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d11189400. 02000000:00000001:1.0:1773941573.499344:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941573.499352:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941573.499356:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c5d8. 00000400:00000010:1.0:1773941573.499363:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000100:00000200:1.0:1773941573.499371:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291491840, portal 25 00000100:00000001:1.0:1773941573.499372:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941573.499374:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00000100:00000040:1.0:1773941573.499378:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c0700 x1860112291491840/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941589 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941573.499384:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941573.499386:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941573.499388:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de908. 00000100:00000200:1.0:1773941573.499391:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291491840, offset 0 00000400:00000200:1.0:1773941573.499397:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941573.499405:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941573.499413:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941573.499419:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941573.499422:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941573.499426:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941573.499432:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941573.499436:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[120, 119] 00000400:00000200:1.0:1773941573.499438:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:1.0:1773941573.499442:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [231:231:256:4294967295] 192.168.204.124@tcp LPNI seq info [121:121:8:4294967295] 00000400:00000200:1.0:1773941573.499446:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941573.499456:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:1.0:1773941573.499460:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:1.0:1773941573.499465:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfd100. 00000800:00000200:1.0:1773941573.499471:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:1.0:1773941573.499479:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:1.0:1773941573.499484:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfd100 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941573.499503:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941573.499507:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941573.499509:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294965095 00000100:00000010:1.0:1773941573.499513:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f31e0. 00000100:00000001:1.0:1773941573.499521:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1773941573.499531:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294965095 00000100:00000010:1.0:1773941573.499535:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3960. 00000100:00000001:1.0:1773941573.499538:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1773941573.499542:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941573.499545:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941573.499548:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941573.499550:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941573.499555:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 11 seconds 00000100:00000001:1.0:1773941573.499559:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941573.499564:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941573.499682:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941573.499691:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfd100. 00000400:00000200:0.0:1773941573.499697:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941573.499704:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941573.499710:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de908 00000400:00000010:0.0:1773941573.499713:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de908. 00000100:00000001:0.0:1773941573.499717:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941573.499724:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c0700 x1860112291491840/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941589 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941573.499733:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941573.499736:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d203c0700 x1860112291491840/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941589 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941573.499742:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941573.499743:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941573.500994:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941573.501006:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941573.501009:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941573.501010:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941573.501017:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941573.501021:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941573.501026:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941573.501030:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941573.501035:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941573.501038:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324607000 00000400:00000200:0.0:1773941573.501042:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941573.501047:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xfc1 [1] + 224 00000800:00000001:0.0:1773941573.501054:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941573.501061:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941573.501063:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941573.501077:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941573.501082:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941573.501085:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c0700 x1860112291491840/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941589 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941573.501093:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d203c0700 x1860112291491840/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941589 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941573.501110:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941573.501117:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941573.501119:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941573.501164:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941573.501175:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941573.501177:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941573.501180:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:1.0:1773941573.501187:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:1.0:1773941573.501189:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:1.0:1773941573.501205:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941573.501213:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c0700 x1860112291491840/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941589 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941573.501225:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c0700 x1860112291491840/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941589 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941573.501233:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941573.501235:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941573.501238:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941573.501241:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941573.501244:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941573.501246:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941573.501249:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941573.501256:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=74 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941573.501259:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941573.501263:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941573.501266:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:1.0:1773941573.501270:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941573.501274:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941573.501277:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941573.501282:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941573.501286:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941573.501290:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941573.501293:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941573.501295:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941573.501297:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941573.501301:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d203c0700 x1860112291491840/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941589 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941573.501308:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941573.501309:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941573.501311:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941573.501314:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941573.501317:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941573.501324:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941573.501327:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941573.501328:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941573.501337:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941573.501339:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941573.501340:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941573.501344:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941573.501346:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941573.501350:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941573.501353:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 294 00000400:00000200:1.0:1773941573.501359:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941573.501364:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941573.501369:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941573.501372:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941573.501375:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941573.501378:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941573.501382:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941573.501384:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941573.501387:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941573.501389:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941573.501392:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941573.501395:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941573.501398:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941573.501401:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941573.501403:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941573.501405:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941573.501409:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941573.501413:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941573.501416:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941573.501419:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941573.501423:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941573.501426:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941573.501434:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941573.501436:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941573.501440:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c3100. 00000020:00000040:1.0:1773941573.501444:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941573.501448:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941573.501450:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941573.501452:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941573.501455:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941573.501458:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941573.501463:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d1118a000. 00000100:00000001:1.0:1773941573.501467:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941573.501473:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c3100 x1860112291491968/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941573.501482:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941573.501485:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941573.501489:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ece65f9] to pc [ptlrpcd_rcv+-1] req@ffff988d203c3100 x1860112291491968/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941573.501513:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941573.501516:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941573.501518:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941573.501521:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941573.501523:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941573.501525:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941573.501527:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941573.501529:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941573.501532:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941573.501537:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c0700 x1860112291491840/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941589 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941573.501544:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941573.501547:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941573.501549:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941573.501551:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941573.501556:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c0700 x1860112291491840/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941589 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941573.501572:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291491840:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941573.501579:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941573.501580:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941573.501583:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941573.501586:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941573.501591:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941573.501591:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941573.501604:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941573.501610:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c3100 x1860112291491968/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941573.501616:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c0700 x1860112291491840/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941589 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941573.501640:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941573.501645:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941573.501655:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1773941573.501657:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941573.501659:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941573.501665:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941573.501665:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941573.501672:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941573.501677:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00100000:3.0:1773941573.501682:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291491968:192.168.204.114@tcp:250:ll_cfg_requeue.0 00010000:00000001:1.0:1773941573.501685:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000001:3.0:1773941573.501687:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00000001:1.0:1773941573.501691:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 02000000:00000001:3.0:1773941573.501692:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941573.501698:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941573.501698:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:3.0:1773941573.501705:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000040:1.0:1773941573.501705:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488800 refcount=4 02000000:00000010:3.0:1773941573.501714:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03520000. 00000020:00000001:1.0:1773941573.501714:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 02000000:00000001:3.0:1773941573.501717:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941573.501726:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00000400:00000200:3.0:1773941573.501727:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941573.501733:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00010000:00010000:1.0:1773941573.501737:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a2b lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:3.0:1773941573.501739:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00000100:00000200:3.0:1773941573.501747:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291491968, portal 25 00000100:00000001:3.0:1773941573.501754:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941573.501760:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631467264 : -113743078084352 : ffff988d203c3100) 00010000:00000001:1.0:1773941573.501760:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000040:3.0:1773941573.501771:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c3100 x1860112291491968/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941583 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:1.0:1773941573.501773:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a2b lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941573.501793:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941573.501817:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:3.0:1773941573.501822:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acbb0. 00010000:00000001:1.0:1773941573.501823:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000200:3.0:1773941573.501827:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291491968, offset 0 00010000:00010000:1.0:1773941573.501830:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a2b lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941573.501837:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941573.501849:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941573.501852:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:3.0:1773941573.501858:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941573.501862:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:1.0:1773941573.501865:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:3.0:1773941573.501869:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00010000:1.0:1773941573.501873:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a2b lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941573.501874:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941573.501882:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941573.501890:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[120, 121] 00000400:00000200:3.0:1773941573.501906:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941573.501914:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [232:232:256:4294967295] 192.168.204.114@tcp LPNI seq info [122:122:8:4294967295] 00010000:00000001:1.0:1773941573.501915:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:3.0:1773941573.501922:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941573.501923:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941573.501930:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:3.0:1773941573.501932:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00010000:00010000:1.0:1773941573.501938:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a2b lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941573.501955:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00010000:00010000:1.0:1773941573.501996:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d01488800 already unused, calling callback (ffffffffc12b4c20) 10000000:00000001:1.0:1773941573.502004:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000800:00000010:3.0:1773941573.502005:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74f00. 10000000:00010000:1.0:1773941573.502015:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a2b lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941573.502018:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:3.0:1773941573.502033:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00010000:00000001:1.0:1773941573.502035:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941573.502038:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000800:00000200:3.0:1773941573.502039:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74f00 type 1, nob 616 niov 1 nkiov 1 00000020:00000001:1.0:1773941573.502041:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941573.502048:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488800 refcount=5 00000020:00000001:1.0:1773941573.502052:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00000100:00000001:3.0:1773941573.502054:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941573.502055:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941573.502062:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:3.0:1773941573.502066:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:1.0:1773941573.502067:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a2b lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941573.502078:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294965098 10000000:00000001:1.0:1773941573.502080:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941573.502083:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a2b lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1773941573.502089:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941573.502091:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000100:00000001:3.0:1773941573.502092:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294965095 10000000:00000001:1.0:1773941573.502094:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941573.502100:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941573.502103:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000100:00000001:3.0:1773941573.502106:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294965098 00010000:00000001:1.0:1773941573.502116:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000100:00000001:3.0:1773941573.502117:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294965095 00010000:00000001:1.0:1773941573.502120:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941573.502122:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941573.502125:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941573.502130:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:0.0:1773941573.502130:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000020:00000040:1.0:1773941573.502131:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d01488800 with handle 0x8e86767622e2a2b from hash 00000800:00000010:0.0:1773941573.502143:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74f00. 00010000:00000001:1.0:1773941573.502145:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000100:00000001:3.0:1773941573.502146:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941573.502151:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000001:1.0:1773941573.502153:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941573.502154:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941573.502158:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:3.0:1773941573.502161:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941573.502161:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00010000:00000001:1.0:1773941573.502164:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941573.502172:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000400:00000200:0.0:1773941573.502172:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acbb0 00000100:00000001:3.0:1773941573.502173:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941573.502178:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000400:00000010:0.0:1773941573.502181:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acbb0. 00010000:00000001:1.0:1773941573.502182:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000100:00000001:3.0:1773941573.502183:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941573.502186:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1773941573.502188:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941573.502194:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941573.502194:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000200:0.0:1773941573.502195:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c3100 x1860112291491968/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941583 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941573.502203:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941573.502207:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941573.502217:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941573.502226:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a2b lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941573.502228:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941573.502236:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c3100 x1860112291491968/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941583 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941573.502241:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941573.502246:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941573.502250:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000100:00000001:3.0:1773941573.502253:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941573.502256:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941573.502263:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:0.0:1773941573.502265:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941573.502266:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941573.502270:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:0.0:1773941573.502271:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00010000:00010000:1.0:1773941573.502275:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a2b lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:1.0:1773941573.502283:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d04dd4e00 count: 0 00010000:00000001:1.0:1773941573.502297:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941573.502298:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941573.502301:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941573.502306:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c0700 x1860112291491840/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941589 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941573.502311:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941573.502314:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941573.502322:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d11189400. 02000000:00000001:1.0:1773941573.502325:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941573.502326:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941573.502328:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941573.502330:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941573.502332:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d181fd800. 02000000:00000001:1.0:1773941573.502335:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941573.502337:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941573.502339:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c0700. 00000100:00000001:1.0:1773941573.502341:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941573.502342:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941573.502344:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941573.502345:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941573.502348:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941573.502352:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d04d67940. 00000020:00000010:1.0:1773941573.502356:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d181fd800. 00000020:00000010:1.0:1773941573.502360:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d181fc400. 00000040:00000010:1.0:1773941573.502364:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d181fd000. 00000020:00000010:1.0:1773941573.502368:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941573.502372:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d181fce00. 80000000:00000010:1.0:1773941573.502376:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d054f3f60. 00040000:00000010:1.0:1773941573.502380:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d181fc800. 20000000:00000010:1.0:1773941573.502385:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d181fc200. 10000000:00000040:1.0:1773941573.502389:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:1.0:1773941573.502391:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941573.502395:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d181fc400. 00000040:00000010:1.0:1773941573.502398:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d181fd000. 00000020:00000010:1.0:1773941573.502400:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941573.502403:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d181fce00. 80000000:00000010:1.0:1773941573.502405:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d054f3f60. 00040000:00000010:1.0:1773941573.502409:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d181fc800. 20000000:00000010:1.0:1773941573.502411:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d181fc200. 00000020:00000010:1.0:1773941573.502414:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d181fd800. 10000000:00000010:1.0:1773941573.502416:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d04d67940. 10000000:01000000:1.0:1773941573.502418:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941573.502425:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941573.502427:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941573.502429:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941573.502430:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941573.502431:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941573.502433:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941573.503528:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941573.503538:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941573.503541:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941573.503543:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941573.503549:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941573.503554:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941573.503560:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941573.503564:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941573.503568:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941573.503580:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324607080 00000400:00000200:0.0:1773941573.503584:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941573.503589:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xfd1 [1] + 0 00000800:00000001:0.0:1773941573.503595:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941573.503601:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941573.503604:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941573.503609:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941573.503613:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941573.503619:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c3100 x1860112291491968/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941583 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941573.503632:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c3100 x1860112291491968/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941583 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941573.503652:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941573.503658:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941573.503662:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941573.503673:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941573.503678:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941573.503684:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941573.503686:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941573.503688:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941573.503693:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941573.503695:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941573.503698:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941573.503705:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c3100 x1860112291491968/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941583 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941573.503716:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c3100 x1860112291491968/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941583 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941573.503737:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941573.503739:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941573.503742:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941573.503745:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941573.503747:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941573.503750:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941573.503753:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941573.503760:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=74 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941573.503765:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941573.503768:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941573.503770:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941573.503773:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941573.503776:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941573.503779:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941573.503784:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c3100 x1860112291491968/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941583 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941573.503793:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941573.503796:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941573.503798:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941573.503819:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941573.503825:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941573.503828:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941573.503831:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941573.503834:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941573.503837:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941573.503840:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941573.503843:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941573.503847:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941573 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941573.503851:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941573.503857:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941573.503860:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941573.503863:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941573.503866:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941573.503875:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941573.503880:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941573.503882:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941573.503885:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941573.503889:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941573.503893:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941573.503896:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941573.504199:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941573.504204:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941573.504211:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c3100 x1860112291491968/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941583 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941573.504219:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291491968:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941573.504222:0:14781:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941573.504226:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941573.504231:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1773941573.504234:0:14781:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:3.0:1773941573.504239:0:14781:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:3.0:1773941573.504243:0:14781:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941573.504246:0:14781:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000040:2.0:1773941573.504247:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c3100 x1860112291491968/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941583 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000001:00000001:3.0:1773941573.504252:0:14781:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941573.504257:0:14781:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941573.504260:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:3.0:1773941573.504263:0:14781:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941573.504265:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:3.0:1773941573.504268:0:14781:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000010:2.0:1773941573.504271:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03520000. 00000001:00000001:3.0:1773941573.504272:0:14781:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941573.504275:0:14781:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941573.504278:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 10000000:00000001:3.0:1773941573.504279:0:14781:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941573.504282:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:3.0:1773941573.504284:0:14781:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000040:2.0:1773941573.504286:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00080000:3.0:1773941573.504290:0:14781:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000020:00000001:2.0:1773941573.504291:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941573.504296:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d1118a000. 00000100:00000001:3.0:1773941573.504297:0:14781:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:3.0:1773941573.504303:0:14781:0:(recover.c:147:ptlrpc_resend()) Process entered 02000000:00000001:2.0:1773941573.504304:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1773941573.504306:0:14781:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941573.504308:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:3.0:1773941573.504312:0:14781:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000010:2.0:1773941573.504313:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c3100. 00000100:00000001:3.0:1773941573.504319:0:14781:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:2.0:1773941573.504320:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 10000000:01000000:3.0:1773941573.504323:0:14781:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000100:00000001:2.0:1773941573.504324:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000040:3.0:1773941573.504327:0:14781:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00000001:2.0:1773941573.504329:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941573.504335:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941573.504337:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941573.504339:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941573.504341:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1773941573.504351:0:14781:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:3.0:1773941573.504355:0:14781:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941573.504357:0:14781:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:3.0:1773941573.504362:0:14781:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:3.0:1773941573.504367:0:14781:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941573.504370:0:14781:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941573.504380:0:14781:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941573.504383:0:14781:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941573.504385:0:14781:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941573.508013:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d04dd4e00. 00010000:00000010:1.1:1773941573.508023:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d01488800. 00000020:00000001:2.0:1773941573.627187:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941573.627195:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941573.627201:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1773941574.651086:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941574.651093:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941574.651098:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941574.779107:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294965098, at 4294965120 00000100:00000040:0.0:1773941574.779107:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294965098, at 4294965120 00000100:00000010:0.0:1773941574.779121:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f31e0. 00000100:00000010:1.0:1773941574.779123:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3960. 10000000:00000001:1.0:1773941574.907031:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941574.907037:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941574.907041:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941574.907043:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941574.907044:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941574.907045:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941574.907047:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941574.907049:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941574.907052:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:1.0:1773941574.907054:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941574.907055:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941574.907064:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c0700. 00000020:00000040:1.0:1773941574.907068:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941574.907072:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941574.907075:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941574.907080:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941574.907086:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d181fc200. 00000100:00000001:1.0:1773941574.907092:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941574.907095:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941574.907097:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941574.907104:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d04dd4e00. 00010000:00000001:1.0:1773941574.907108:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941574.907111:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d01488800. 00000020:00000001:1.0:1773941574.907114:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941574.907116:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d01488800 with handle 0x8e86767622e2a32 to hash 00000020:00000001:1.0:1773941574.907118:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941574.907119:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941574.907121:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941574.907135:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941574.907138:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941574.907141:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a32 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941574.907153:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a32 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941574.907165:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a32 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941574.907172:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941574.907174:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941574.907181:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941574.907184:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330528610752 : -113743180940864 : ffff988d1a1ab9c0) 00000100:00000001:1.0:1773941574.907187:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941574.907189:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00000100:00000001:1.0:1773941574.907193:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941574.907198:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941574.907200:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941574.907202:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941574.907213:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c0700 x1860112291492096/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941574.907222:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941574.907224:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941574.907228:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941574.907231:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941574.907233:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941574.907237:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291492096:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941574.907242:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941574.907247:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941574.907249:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941574.907252:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941574.907262:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d11189400. 02000000:00000001:1.0:1773941574.907263:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941574.907271:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941574.907275:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c5d8. 00000400:00000010:1.0:1773941574.907282:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000100:00000200:1.0:1773941574.907289:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291492096, portal 25 00000100:00000001:1.0:1773941574.907291:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941574.907292:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00000100:00000040:1.0:1773941574.907308:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c0700 x1860112291492096/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941590 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941574.907319:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941574.907322:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941574.907325:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113deaa0. 00000100:00000200:1.0:1773941574.907328:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291492096, offset 0 00000400:00000200:1.0:1773941574.907335:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941574.907343:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941574.907351:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941574.907356:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941574.907360:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941574.907365:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941574.907367:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941574.907371:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[122, 121] 00000400:00000200:1.0:1773941574.907374:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:1.0:1773941574.907379:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [233:233:256:4294967295] 192.168.204.124@tcp LPNI seq info [123:123:8:4294967295] 00000400:00000200:1.0:1773941574.907383:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941574.907394:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:1.0:1773941574.907400:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:1.0:1773941574.907405:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfd600. 00000800:00000200:1.0:1773941574.907412:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:1.0:1773941574.907421:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:1.0:1773941574.907425:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfd600 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941574.907450:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941574.907455:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941574.907458:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294966503 00000100:00000010:1.0:1773941574.907462:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3960. 00000100:00000001:1.0:1773941574.907469:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1773941574.907477:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294966503 00000100:00000010:1.0:1773941574.907481:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3f60. 00000100:00000001:1.0:1773941574.907484:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1773941574.907487:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941574.907489:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941574.907492:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941574.907493:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941574.907496:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 11 seconds 00000100:00000001:1.0:1773941574.907498:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941574.907500:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941574.907597:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941574.907604:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfd600. 00000400:00000200:0.0:1773941574.907610:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941574.907616:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941574.907622:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113deaa0 00000400:00000010:0.0:1773941574.907625:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113deaa0. 00000100:00000001:0.0:1773941574.907631:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941574.907637:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c0700 x1860112291492096/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941590 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941574.907648:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941574.907652:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d203c0700 x1860112291492096/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941590 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941574.907660:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941574.907662:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941574.908988:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941574.909001:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941574.909004:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941574.909005:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941574.909010:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941574.909013:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941574.909018:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941574.909021:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941574.909024:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941574.909027:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324607100 00000400:00000200:0.0:1773941574.909030:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941574.909034:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xfd9 [1] + 224 00000800:00000001:0.0:1773941574.909038:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941574.909045:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941574.909047:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941574.909049:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941574.909052:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941574.909056:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c0700 x1860112291492096/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941590 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941574.909063:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d203c0700 x1860112291492096/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941590 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941574.909077:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941574.909083:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941574.909086:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941574.909117:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941574.909123:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941574.909125:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941574.909128:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:1.0:1773941574.909132:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:1.0:1773941574.909134:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:1.0:1773941574.909137:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941574.909143:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c0700 x1860112291492096/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941590 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941574.909153:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c0700 x1860112291492096/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941590 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941574.909161:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941574.909163:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941574.909166:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941574.909168:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941574.909171:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941574.909174:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941574.909176:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941574.909183:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=75 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941574.909193:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941574.909197:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941574.909201:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:1.0:1773941574.909204:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941574.909208:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941574.909211:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941574.909215:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941574.909220:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941574.909224:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941574.909227:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941574.909229:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941574.909231:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941574.909235:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d203c0700 x1860112291492096/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941590 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941574.909244:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941574.909246:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941574.909247:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941574.909250:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941574.909253:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941574.909255:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941574.909257:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941574.909258:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941574.909260:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941574.909261:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941574.909263:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941574.909265:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941574.909267:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941574.909271:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941574.909273:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 296 00000400:00000200:1.0:1773941574.909278:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941574.909283:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941574.909288:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941574.909291:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941574.909313:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941574.909315:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941574.909319:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941574.909320:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941574.909322:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941574.909325:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941574.909327:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941574.909330:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941574.909333:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941574.909335:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941574.909337:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941574.909339:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941574.909342:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941574.909346:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941574.909349:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941574.909351:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941574.909354:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941574.909357:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941574.909360:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941574.909362:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941574.909366:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c0a80. 00000020:00000040:1.0:1773941574.909369:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941574.909373:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941574.909375:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941574.909377:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941574.909379:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941574.909382:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941574.909386:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d11189000. 00000100:00000001:1.0:1773941574.909391:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941574.909397:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c0a80 x1860112291492224/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941574.909405:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941574.909413:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941574.909419:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000431867b8] to pc [ptlrpcd_rcv+-1] req@ffff988d203c0a80 x1860112291492224/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941574.909445:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941574.909447:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941574.909449:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941574.909452:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941574.909454:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941574.909456:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941574.909458:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941574.909460:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941574.909462:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941574.909467:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c0700 x1860112291492096/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941590 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941574.909475:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941574.909479:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941574.909481:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941574.909483:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941574.909488:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c0700 x1860112291492096/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941590 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941574.909492:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941574.909497:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1773941574.909501:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291492096:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941574.909502:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941574.909509:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941574.909513:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c0a80 x1860112291492224/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941574.909513:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941574.909516:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941574.909520:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941574.909527:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c0700 x1860112291492096/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941590 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941574.909529:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941574.909532:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941574.909538:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941574.909542:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941574.909543:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941574.909547:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941574.909548:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941574.909552:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00100000:2.0:1773941574.909561:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291492224:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941574.909562:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:1.0:1773941574.909567:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000001:2.0:1773941574.909569:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00000001:1.0:1773941574.909572:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 02000000:00000001:2.0:1773941574.909574:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:1.0:1773941574.909576:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:2.0:1773941574.909578:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941574.909581:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488800 refcount=4 02000000:00000001:2.0:1773941574.909582:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:1.0:1773941574.909587:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 02000000:00000010:2.0:1773941574.909589:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a5c00. 00010000:00000001:1.0:1773941574.909592:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 02000000:00000001:2.0:1773941574.909594:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941574.909601:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:1.0:1773941574.909601:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a32 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941574.909607:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000400:00000010:2.0:1773941574.909614:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00010000:00000001:1.0:1773941574.909615:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000200:2.0:1773941574.909620:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291492224, portal 25 00010000:00010000:1.0:1773941574.909622:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a32 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941574.909624:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941574.909627:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631457408 : -113743078094208 : ffff988d203c0a80) 00000100:00000040:2.0:1773941574.909637:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c0a80 x1860112291492224/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941584 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941574.909638:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941574.909643:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a32 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941574.909657:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:1.0:1773941574.909660:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000040:2.0:1773941574.909664:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00000001:1.0:1773941574.909664:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941574.909668:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a32 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941574.909670:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516220. 00000100:00000200:2.0:1773941574.909677:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291492224, offset 0 00010000:00000001:1.0:1773941574.909682:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941574.909685:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941574.909688:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00000001:1.0:1773941574.909690:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941574.909695:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a32 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941574.909696:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941574.909705:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00010000:1.0:1773941574.909709:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d01488800 already unused, calling callback (ffffffffc12b4c20) 00000400:00000200:2.0:1773941574.909711:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 10000000:00000001:1.0:1773941574.909715:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:2.0:1773941574.909717:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 10000000:00010000:1.0:1773941574.909720:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a32 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941574.909721:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941574.909725:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941574.909730:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[122, 123] 00000400:00000200:2.0:1773941574.909742:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00010000:00000001:1.0:1773941574.909745:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941574.909749:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:2.0:1773941574.909750:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [234:234:256:4294967295] 192.168.204.114@tcp LPNI seq info [124:124:8:4294967295] 00000020:00000001:1.0:1773941574.909752:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941574.909755:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488800 refcount=5 00000400:00000200:2.0:1773941574.909760:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000020:00000001:1.0:1773941574.909761:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941574.909766:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941574.909771:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941574.909789:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a32 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941574.909791:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 10000000:00000001:1.0:1773941574.909802:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941574.909806:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a32 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941574.909817:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 10000000:01000000:1.0:1773941574.909818:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941574.909823:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000800:00000010:2.0:1773941574.909824:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0dbb00. 10000000:00000001:1.0:1773941574.909826:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941574.909832:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000800:00000200:2.0:1773941574.909834:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00010000:00000001:1.0:1773941574.909835:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941574.909842:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000800:00000200:2.0:1773941574.909846:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00010000:00000001:1.0:1773941574.909846:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941574.909850:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000800:00000200:2.0:1773941574.909852:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0dbb00 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941574.909853:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941574.909859:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d01488800 with handle 0x8e86767622e2a32 from hash 00010000:00000001:1.0:1773941574.909862:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941574.909863:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941574.909864:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941574.909866:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941574.909868:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000100:00000001:2.0:1773941574.909869:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941574.909870:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941574.909876:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:2.0:1773941574.909877:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941574.909878:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941574.909883:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294966505 00010000:00000001:1.0:1773941574.909883:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941574.909891:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294966503 10000000:00000001:1.0:1773941574.909893:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941574.909898:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294966505 00010000:00010000:1.0:1773941574.909900:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a32 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941574.909906:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294966503 00010000:00000001:1.0:1773941574.909910:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941574.909913:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941574.909914:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941574.909918:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000100:00000001:2.0:1773941574.909919:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941574.909920:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941574.909924:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941574.909925:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941574.909926:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941574.909930:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941574.909931:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:1.0:1773941574.909933:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a32 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:0.0:1773941574.909935:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1773941574.909939:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000800:00000010:0.0:1773941574.909940:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0dbb00. 00000100:00000001:2.0:1773941574.909943:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000400:00000200:0.0:1773941574.909947:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:2.0:1773941574.909950:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000040:1.0:1773941574.909952:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d04dd4e00 count: 0 00000100:00000001:2.0:1773941574.909954:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000200:0.0:1773941574.909956:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:2.0:1773941574.909987:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941574.909993:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516220 00000100:00000001:2.0:1773941574.909994:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:0.0:1773941574.909997:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516220. 00010000:00000001:1.0:1773941574.910002:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1773941574.910003:0:2393:0:(events.c:34:request_out_callback()) Process entered 00010000:00000001:1.0:1773941574.910006:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000200:0.0:1773941574.910010:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c0a80 x1860112291492224/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941584 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941574.910012:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941574.910021:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c0700 x1860112291492096/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941590 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941574.910048:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1773941574.910051:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1773941574.910052:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941574.910057:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d11189400. 00000100:00000040:0.0:1773941574.910058:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c0a80 x1860112291492224/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941584 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1773941574.910064:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1773941574.910067:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941574.910068:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:0.0:1773941574.910070:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000040:1.0:1773941574.910072:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941574.910077:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941574.910079:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d181fc200. 02000000:00000001:1.0:1773941574.910084:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941574.910086:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941574.910088:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c0700. 00000100:00000001:1.0:1773941574.910092:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941574.910093:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941574.910095:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941574.910098:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941574.910101:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941574.910112:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d04d67d40. 00000020:00000010:1.0:1773941574.910116:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d181fc200. 00000020:00000010:1.0:1773941574.910120:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d181fc800. 00000040:00000010:1.0:1773941574.910124:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d181fce00. 00000020:00000010:1.0:1773941574.910129:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941574.910134:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d181fd000. 80000000:00000010:1.0:1773941574.910138:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d054f3a80. 00040000:00000010:1.0:1773941574.910142:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d181fc400. 20000000:00000010:1.0:1773941574.910147:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d181fd800. 10000000:00000040:1.0:1773941574.910153:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:1.0:1773941574.910155:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941574.910160:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d181fc800. 00000040:00000010:1.0:1773941574.910163:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d181fce00. 00000020:00000010:1.0:1773941574.910166:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941574.910170:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d181fd000. 80000000:00000010:1.0:1773941574.910173:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d054f3a80. 00040000:00000010:1.0:1773941574.910177:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d181fc400. 20000000:00000010:1.0:1773941574.910180:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d181fd800. 00000020:00000010:1.0:1773941574.910183:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d181fc200. 10000000:00000010:1.0:1773941574.910186:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d04d67d40. 10000000:01000000:1.0:1773941574.910189:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941574.910192:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941574.910194:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941574.910196:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941574.910198:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941574.910199:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941574.910202:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941574.911328:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941574.911337:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941574.911339:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941574.911341:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941574.911345:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941574.911348:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941574.911353:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941574.911356:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941574.911358:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941574.911361:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324607180 00000400:00000200:0.0:1773941574.911364:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941574.911367:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0xfe9 [1] + 0 00000800:00000001:0.0:1773941574.911372:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941574.911378:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941574.911379:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941574.911382:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941574.911386:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941574.911391:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c0a80 x1860112291492224/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941584 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941574.911399:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c0a80 x1860112291492224/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941584 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941574.911413:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941574.911417:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941574.911419:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941574.911447:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941574.911451:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941574.911455:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941574.911456:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941574.911458:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941574.911462:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941574.911464:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941574.911467:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941574.911473:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c0a80 x1860112291492224/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941584 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941574.911482:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c0a80 x1860112291492224/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941584 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941574.911501:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941574.911503:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941574.911505:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941574.911508:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941574.911509:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941574.911512:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941574.911514:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941574.911518:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=75 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941574.911522:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941574.911524:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941574.911526:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941574.911528:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941574.911530:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941574.911533:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941574.911537:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c0a80 x1860112291492224/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941584 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941574.911544:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941574.911547:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941574.911548:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941574.911550:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941574.911555:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941574.911558:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941574.911559:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941574.911561:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941574.911563:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941574.911565:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941574.911568:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:2.0:1773941574.911572:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941574 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941574.911575:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941574.911582:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:2.0:1773941574.911584:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941574.911585:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941574.911587:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941574.911594:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941574.911598:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:2.0:1773941574.911599:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941574.911601:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941574.911604:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941574.911606:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:2.0:1773941574.911608:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941574.911870:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941574.911883:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941574.911891:0:14782:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000040:3.0:1773941574.911899:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c0a80 x1860112291492224/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941584 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1773941574.911904:0:14782:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:2.0:1773941574.911913:0:14782:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00100000:3.0:1773941574.911933:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291492224:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941574.911946:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941574.911950:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941574.911956:0:14782:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000040:3.0:1773941574.911957:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c0a80 x1860112291492224/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941584 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:01000000:2.0:1773941574.911975:0:14782:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:2.0:1773941574.911979:0:14782:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941574.911982:0:14782:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941574.911986:0:14782:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:2.0:1773941574.911989:0:14782:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941574.911990:0:14782:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941574.911993:0:14782:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941574.911997:0:14782:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941574.912005:0:14782:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941574.912012:0:14782:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:3.0:1773941574.912013:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1773941574.912017:0:14782:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 02000000:00000001:3.0:1773941574.912019:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1773941574.912021:0:14782:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:2.0:1773941574.912024:0:14782:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941574.912025:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a5c00. 00000100:00080000:2.0:1773941574.912035:0:14782:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 02000000:00000001:3.0:1773941574.912037:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1773941574.912039:0:14782:0:(obd_class.h:1215:obd_import_event()) Process entered 00000020:00000001:3.0:1773941574.912040:0:2400:0:(genops.c:1236:class_import_put()) Process entered 10000000:01000000:2.0:1773941574.912041:0:14782:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000020:00000040:3.0:1773941574.912043:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 10000000:00000040:2.0:1773941574.912044:0:14782:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000020:00000001:3.0:1773941574.912051:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941574.912054:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d11189000. 02000000:00000001:3.0:1773941574.912059:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941574.912060:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941574.912062:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c0a80. 00000100:00080000:2.0:1773941574.912062:0:14782:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:2.0:1773941574.912067:0:14782:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941574.912068:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941574.912070:0:14782:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941574.912072:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941574.912077:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:02000000:2.0:1773941574.912078:0:14782:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:2.0:1773941574.912089:0:14782:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941574.912091:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:2.0:1773941574.912093:0:14782:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:3.0:1773941574.912095:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941574.912097:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941574.912105:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941574.912105:0:14782:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941574.912116:0:14782:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941574.912118:0:14782:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941574.918001:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d04dd4e00. 00010000:00000010:1.1:1773941574.918016:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d01488800. 00000020:00000001:2.0:1773941575.675118:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941575.675124:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941575.675129:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1773941576.187043:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294966505, at 4294966528 00000100:00000040:1.0:1773941576.187053:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294966505, at 4294966528 00000100:00000010:1.0:1773941576.187070:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3f60. 00000100:00000010:0.0:1773941576.187072:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3960. 00000020:00000001:2.0:1773941576.699099:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941576.699106:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941576.699110:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941576.891114:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941576.891121:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941576.891127:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941576.891129:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941576.891149:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941576.891151:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941576.891154:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941576.891156:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941576.891159:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:1.0:1773941576.891162:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941576.891164:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941576.891171:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c0700. 00000020:00000040:1.0:1773941576.891176:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941576.891180:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941576.891183:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941576.891187:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941576.891193:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d181fd800. 00000100:00000001:1.0:1773941576.891198:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941576.891202:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941576.891205:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941576.891212:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d04dd4e00. 00010000:00000001:1.0:1773941576.891217:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941576.891219:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d01488800. 00000020:00000001:1.0:1773941576.891224:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941576.891227:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d01488800 with handle 0x8e86767622e2a39 to hash 00000020:00000001:1.0:1773941576.891229:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941576.891231:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941576.891235:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941576.891238:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941576.891243:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941576.891247:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a39 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941576.891258:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a39 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941576.891287:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a39 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941576.891302:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941576.891304:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941576.891309:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941576.891311:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330528610752 : -113743180940864 : ffff988d1a1ab9c0) 00000100:00000001:1.0:1773941576.891313:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941576.891315:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00000100:00000001:1.0:1773941576.891318:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941576.891322:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941576.891324:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941576.891326:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941576.891334:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c0700 x1860112291492352/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941576.891341:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941576.891343:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941576.891347:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941576.891349:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941576.891350:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941576.891353:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291492352:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941576.891356:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941576.891360:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941576.891362:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941576.891364:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941576.891368:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d11189400. 02000000:00000001:1.0:1773941576.891370:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941576.891376:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941576.891379:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c5d8. 00000400:00000010:1.0:1773941576.891384:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000100:00000200:1.0:1773941576.891390:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291492352, portal 25 00000100:00000001:1.0:1773941576.891391:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941576.891392:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00000100:00000040:1.0:1773941576.891396:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c0700 x1860112291492352/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941592 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941576.891411:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941576.891414:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941576.891415:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de2a8. 00000100:00000200:1.0:1773941576.891418:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291492352, offset 0 00000400:00000200:1.0:1773941576.891423:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941576.891430:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941576.891436:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941576.891440:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941576.891444:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941576.891447:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941576.891449:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941576.891452:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[124, 123] 00000400:00000200:1.0:1773941576.891454:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:1.0:1773941576.891457:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [235:235:256:4294967295] 192.168.204.124@tcp LPNI seq info [125:125:8:4294967295] 00000400:00000200:1.0:1773941576.891460:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941576.891468:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:1.0:1773941576.891472:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:1.0:1773941576.891477:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfd600. 00000800:00000200:1.0:1773941576.891483:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:1.0:1773941576.891489:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:1.0:1773941576.891492:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfd600 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941576.891506:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941576.891509:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941576.891511:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294968487 00000100:00000010:1.0:1773941576.891515:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3f60. 00000100:00000001:1.0:1773941576.891520:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1773941576.891524:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294968487 00000100:00000010:1.0:1773941576.891526:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3a80. 00000100:00000001:1.0:1773941576.891528:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1773941576.891534:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941576.891536:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941576.891538:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941576.891540:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941576.891542:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 11 seconds 00000100:00000001:1.0:1773941576.891544:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941576.891546:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941576.891649:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941576.891654:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfd600. 00000400:00000200:0.0:1773941576.891658:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941576.891663:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941576.891668:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de2a8 00000400:00000010:0.0:1773941576.891669:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de2a8. 00000100:00000001:0.0:1773941576.891673:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941576.891677:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c0700 x1860112291492352/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941592 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941576.891685:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941576.891688:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d203c0700 x1860112291492352/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941592 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941576.891693:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941576.891695:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941576.892949:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941576.892987:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941576.892991:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941576.892995:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941576.893000:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941576.893006:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941576.893012:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941576.893017:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941576.893023:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941576.893028:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324607200 00000400:00000200:0.0:1773941576.893033:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941576.893039:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0xff1 [1] + 224 00000800:00000001:0.0:1773941576.893046:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941576.893054:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941576.893057:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941576.893060:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941576.893065:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941576.893078:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c0700 x1860112291492352/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941592 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941576.893088:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d203c0700 x1860112291492352/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941592 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941576.893106:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941576.893114:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941576.893117:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941576.893140:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941576.893147:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941576.893149:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941576.893151:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:1.0:1773941576.893155:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:1.0:1773941576.893157:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:1.0:1773941576.893159:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941576.893164:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c0700 x1860112291492352/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941592 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941576.893174:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c0700 x1860112291492352/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941592 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941576.893181:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941576.893183:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941576.893185:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941576.893187:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941576.893189:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941576.893192:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941576.893194:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941576.893200:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=77 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941576.893203:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941576.893207:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941576.893210:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:1.0:1773941576.893213:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941576.893217:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941576.893220:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941576.893224:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941576.893229:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941576.893238:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941576.893240:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941576.893242:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941576.893244:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941576.893248:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d203c0700 x1860112291492352/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941592 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941576.893256:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941576.893258:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941576.893260:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941576.893263:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941576.893265:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941576.893267:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941576.893269:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941576.893270:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941576.893271:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941576.893272:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941576.893274:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941576.893276:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941576.893278:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941576.893281:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941576.893283:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 297 00000400:00000200:1.0:1773941576.893288:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941576.893292:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941576.893295:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941576.893299:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941576.893302:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941576.893304:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941576.893307:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941576.893309:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941576.893311:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941576.893313:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941576.893316:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941576.893318:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941576.893327:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941576.893329:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941576.893332:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941576.893334:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941576.893337:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941576.893340:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941576.893342:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941576.893345:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941576.893348:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941576.893350:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941576.893353:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941576.893355:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941576.893359:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c1500. 00000020:00000040:1.0:1773941576.893362:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941576.893365:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941576.893367:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941576.893369:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941576.893371:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941576.893373:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941576.893377:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d11188000. 00000100:00000001:1.0:1773941576.893381:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941576.893387:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c1500 x1860112291492480/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941576.893395:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941576.893398:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941576.893402:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000022b3efad] to pc [ptlrpcd_rcv+-1] req@ffff988d203c1500 x1860112291492480/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941576.893423:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941576.893427:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941576.893429:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941576.893431:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941576.893434:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941576.893441:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941576.893444:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941576.893447:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941576.893449:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941576.893456:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c0700 x1860112291492352/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941592 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941576.893459:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941576.893463:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941576.893466:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941576.893470:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000040:3.0:1773941576.893474:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c1500 x1860112291492480/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941576.893477:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941576.893482:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941576.893485:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941576.893493:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c0700 x1860112291492352/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941592 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941576.893507:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291492352:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941576.893516:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941576.893520:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941576.893521:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941576.893525:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:3.0:1773941576.893526:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941576.893527:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941576.893532:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c0700 x1860112291492352/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941592 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1773941576.893533:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941576.893537:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941576.893540:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941576.893543:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941576.893544:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941576.893550:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941576.893552:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00100000:3.0:1773941576.893555:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291492480:192.168.204.114@tcp:250:ll_cfg_requeue.0 00010000:00000001:1.0:1773941576.893555:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941576.893559:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941576.893562:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:3.0:1773941576.893565:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000020:00000040:1.0:1773941576.893565:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488800 refcount=4 00000020:00000001:1.0:1773941576.893569:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 02000000:00000001:3.0:1773941576.893570:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00000001:1.0:1773941576.893572:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 02000000:00000001:3.0:1773941576.893574:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941576.893576:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a39 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941576.893578:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941576.893582:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03522400. 02000000:00000001:3.0:1773941576.893586:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941576.893594:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941576.893595:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000010:3.0:1773941576.893597:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00000400:00000010:3.0:1773941576.893604:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00010000:00010000:1.0:1773941576.893604:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a39 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:3.0:1773941576.893607:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291492480, portal 25 00000100:00000001:3.0:1773941576.893611:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941576.893613:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631460096 : -113743078091520 : ffff988d203c1500) 00000100:00000040:3.0:1773941576.893621:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c1500 x1860112291492480/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941586 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941576.893622:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941576.893628:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a39 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941576.893629:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941576.893634:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:3.0:1773941576.893638:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acd48. 00000100:00000200:3.0:1773941576.893642:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291492480, offset 0 00010000:00000001:1.0:1773941576.893645:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:3.0:1773941576.893649:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00000001:1.0:1773941576.893650:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:3.0:1773941576.893654:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:1.0:1773941576.893655:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a39 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941576.893659:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941576.893666:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941576.893670:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00000001:1.0:1773941576.893675:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:3.0:1773941576.893676:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941576.893681:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941576.893682:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941576.893687:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:3.0:1773941576.893691:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[124, 125] 00010000:00010000:1.0:1773941576.893693:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a39 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941576.893696:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941576.893702:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [236:236:256:4294967295] 192.168.204.114@tcp LPNI seq info [126:126:8:4294967295] 00000400:00000200:3.0:1773941576.893713:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:1.0:1773941576.893714:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d01488800 already unused, calling callback (ffffffffc12b4c20) 10000000:00000001:1.0:1773941576.893719:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:3.0:1773941576.893721:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 10000000:00010000:1.0:1773941576.893724:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a39 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941576.893726:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:3.0:1773941576.893732:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74500. 00000800:00000200:3.0:1773941576.893753:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00010000:00000001:1.0:1773941576.893758:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000800:00000200:3.0:1773941576.893762:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00010000:00000001:1.0:1773941576.893763:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000800:00000200:3.0:1773941576.893766:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74500 type 1, nob 616 niov 1 nkiov 1 00000020:00000001:1.0:1773941576.893767:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941576.893772:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488800 refcount=5 00000020:00000001:1.0:1773941576.893775:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941576.893778:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00000100:00000001:3.0:1773941576.893782:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941576.893782:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:3.0:1773941576.893789:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:3.0:1773941576.893794:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294968489 00010000:00010000:1.0:1773941576.893796:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a39 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941576.893798:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294968487 00000100:00000001:3.0:1773941576.893803:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294968489 00000100:00000001:3.0:1773941576.893810:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294968487 10000000:00000001:1.0:1773941576.893811:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000001:3.0:1773941576.893815:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 10000000:00010000:1.0:1773941576.893817:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a39 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941576.893818:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941576.893823:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941576.893828:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:01000000:1.0:1773941576.893832:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000100:00000001:3.0:1773941576.893836:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 10000000:00000040:1.0:1773941576.893837:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000100:00000001:3.0:1773941576.893839:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 10000000:00000001:1.0:1773941576.893841:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941576.893842:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941576.893845:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941576.893848:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941576.893848:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000100:00000001:3.0:1773941576.893853:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941576.893854:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941576.893859:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000800:00000200:0.0:1773941576.893860:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00000001:1.0:1773941576.893862:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000800:00000010:0.0:1773941576.893865:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74500. 00010000:00000001:1.0:1773941576.893867:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:0.0:1773941576.893870:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000001:1.0:1773941576.893872:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941576.893877:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d01488800 with handle 0x8e86767622e2a39 from hash 00000400:00000200:0.0:1773941576.893878:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00010000:00000001:1.0:1773941576.893884:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000400:00000200:0.0:1773941576.893885:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acd48 00010000:00000001:1.0:1773941576.893890:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000010:0.0:1773941576.893890:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acd48. 00010000:00000001:1.0:1773941576.893895:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1773941576.893896:0:2393:0:(events.c:34:request_out_callback()) Process entered 00010000:00000001:1.0:1773941576.893899:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000100:00000200:0.0:1773941576.893902:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c1500 x1860112291492480/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941586 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941576.893903:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941576.893907:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941576.893914:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:0.0:1773941576.893915:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00010000:00000001:1.0:1773941576.893918:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000040:0.0:1773941576.893920:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c1500 x1860112291492480/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941586 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941576.893922:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941576.893927:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941576.893934:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a39 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941576.893934:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941576.893937:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:1.0:1773941576.893944:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941576.893946:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941576.893948:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941576.893950:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941576.893952:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941576.893953:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941576.893987:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941576.893990:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a39 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:1.0:1773941576.893999:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d04dd4e00 count: 0 00010000:00000001:1.0:1773941576.894009:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941576.894011:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941576.894014:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941576.894020:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c0700 x1860112291492352/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941592 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941576.894028:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941576.894031:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941576.894034:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d11189400. 02000000:00000001:1.0:1773941576.894038:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941576.894039:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941576.894041:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941576.894044:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941576.894046:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d181fd800. 02000000:00000001:1.0:1773941576.894050:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941576.894052:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941576.894053:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c0700. 00000100:00000001:1.0:1773941576.894057:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941576.894058:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941576.894060:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941576.894063:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941576.894066:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941576.894070:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d04d67400. 00000020:00000010:1.0:1773941576.894074:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d181fd800. 00000020:00000010:1.0:1773941576.894079:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d181fc400. 00000040:00000010:1.0:1773941576.894083:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d181fd000. 00000020:00000010:1.0:1773941576.894088:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941576.894093:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d181fce00. 80000000:00000010:1.0:1773941576.894097:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d054f3840. 00040000:00000010:1.0:1773941576.894101:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d181fc800. 20000000:00000010:1.0:1773941576.894106:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d181fd600. 10000000:00000040:1.0:1773941576.894116:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:1.0:1773941576.894118:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941576.894123:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d181fc400. 00000040:00000010:1.0:1773941576.894127:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d181fd000. 00000020:00000010:1.0:1773941576.894130:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941576.894134:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d181fce00. 80000000:00000010:1.0:1773941576.894138:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d054f3840. 00040000:00000010:1.0:1773941576.894142:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d181fc800. 20000000:00000010:1.0:1773941576.894145:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d181fd600. 00000020:00000010:1.0:1773941576.894148:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d181fd800. 10000000:00000010:1.0:1773941576.894150:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d04d67400. 10000000:01000000:1.0:1773941576.894153:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941576.894156:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941576.894159:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941576.894161:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941576.894162:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941576.894163:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941576.894166:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941576.895539:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941576.895545:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941576.895547:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941576.895549:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941576.895552:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941576.895556:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941576.895560:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941576.895563:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941576.895567:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941576.895571:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324607280 00000400:00000200:0.0:1773941576.895575:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941576.895589:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0x1001 [1] + 0 00000800:00000001:0.0:1773941576.895595:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941576.895602:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941576.895605:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941576.895609:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941576.895612:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941576.895617:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c1500 x1860112291492480/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941586 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941576.895629:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c1500 x1860112291492480/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941586 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941576.895645:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941576.895651:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941576.895654:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941576.895671:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941576.895674:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941576.895678:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941576.895679:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941576.895682:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941576.895686:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941576.895693:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941576.895695:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941576.895698:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c1500 x1860112291492480/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941586 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941576.895705:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c1500 x1860112291492480/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941586 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941576.895709:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941576.895710:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941576.895711:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941576.895713:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941576.895714:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941576.895715:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941576.895717:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941576.895720:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=77 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941576.895722:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941576.895724:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941576.895725:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941576.895727:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941576.895728:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941576.895730:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941576.895732:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c1500 x1860112291492480/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941586 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941576.895748:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941576.895750:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941576.895751:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941576.895752:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941576.895756:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941576.895758:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941576.895759:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941576.895760:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941576.895762:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941576.895763:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941576.895765:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941576.895768:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941576 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941576.895775:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941576.895780:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941576.895782:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941576.895783:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941576.895784:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941576.895786:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941576.895788:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941576.895790:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941576.895792:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941576.895795:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941576.895797:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941576.895799:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941576.895995:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941576.896000:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941576.896007:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c1500 x1860112291492480/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941586 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941576.896017:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291492480:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:0.0:1773941576.896020:0:14783:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:3.0:1773941576.896023:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941576.896025:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941576.896028:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c1500 x1860112291492480/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941586 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1773941576.896028:0:14783:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:0.0:1773941576.896032:0:14783:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:0.0:1773941576.896034:0:14783:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:3.0:1773941576.896035:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 10000000:01000000:0.0:1773941576.896036:0:14783:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000001:3.0:1773941576.896037:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941576.896039:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03522400. 00000001:00000001:0.0:1773941576.896039:0:14783:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:3.0:1773941576.896042:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:0.0:1773941576.896042:0:14783:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000001:3.0:1773941576.896043:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941576.896045:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941576.896047:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:0.0:1773941576.896047:0:14783:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941576.896049:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d11188000. 00000001:00000001:0.0:1773941576.896049:0:14783:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941576.896051:0:14783:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:3.0:1773941576.896053:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000001:00000001:0.0:1773941576.896053:0:14783:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941576.896055:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 10000000:00000001:0.0:1773941576.896056:0:14783:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941576.896057:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c1500. 00000100:00000001:0.0:1773941576.896058:0:14783:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:0.0:1773941576.896062:0:14783:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:0.0:1773941576.896065:0:14783:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:3.0:1773941576.896066:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941576.896067:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941576.896067:0:14783:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:0.0:1773941576.896069:0:14783:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941576.896070:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1773941576.896072:0:14783:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:3.0:1773941576.896073:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941576.896075:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941576.896075:0:14783:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941576.896076:0:14783:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000100:00000001:3.0:1773941576.896077:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 10000000:00000040:0.0:1773941576.896078:0:14783:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00000001:3.0:1773941576.896079:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941576.896100:0:14783:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941576.896103:0:14783:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941576.896105:0:14783:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:0.0:1773941576.896111:0:14783:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:0.0:1773941576.896114:0:14783:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941576.896117:0:14783:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941576.896118:0:14783:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:0.0:1773941576.896120:0:14783:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941576.896122:0:14783:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941576.899997:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d04dd4e00. 00010000:00000010:1.1:1773941576.900007:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d01488800. 00000100:00000040:0.0:1773941576.955037:0:8:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (304) 00000020:00000001:2.0:1773941577.723176:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941577.723185:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941577.723190:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941578.171147:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294968489, at 4294968512 00000100:00000040:0.0:1773941578.171150:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294968489, at 4294968512 00000100:00000010:0.0:1773941578.171179:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3f60. 00000100:00000010:1.0:1773941578.171183:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3a80. 10000000:00000001:1.0:1773941578.683143:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941578.683151:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941578.683157:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941578.683160:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941578.683162:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941578.683164:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941578.683167:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941578.683170:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941578.683173:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:1.0:1773941578.683176:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941578.683178:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941578.683192:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c0700. 00000020:00000040:1.0:1773941578.683197:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941578.683202:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941578.683223:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941578.683229:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941578.683237:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d181fc800. 00000100:00000001:1.0:1773941578.683244:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941578.683249:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941578.683252:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941578.683260:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d04dd4e00. 00010000:00000001:1.0:1773941578.683266:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941578.683269:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d01488800. 00000020:00000001:1.0:1773941578.683274:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941578.683276:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d01488800 with handle 0x8e86767622e2a40 to hash 00000020:00000001:1.0:1773941578.683278:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941578.683279:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941578.683282:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941578.683285:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941578.683288:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941578.683291:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a40 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941578.683300:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a40 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941578.683310:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a40 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941578.683315:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941578.683316:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941578.683322:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941578.683324:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330528610752 : -113743180940864 : ffff988d1a1ab9c0) 00000100:00000001:1.0:1773941578.683326:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941578.683328:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00000100:00000001:1.0:1773941578.683333:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941578.683338:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941578.683339:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941578.683346:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941578.683358:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c0700 x1860112291492608/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941578.683366:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941578.683368:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941578.683371:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941578.683373:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941578.683375:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941578.683378:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291492608:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941578.683381:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941578.683388:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941578.683390:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941578.683392:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941578.683396:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d11189400. 02000000:00000001:1.0:1773941578.683398:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941578.683407:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941578.683410:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c5d8. 00000400:00000010:1.0:1773941578.683416:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000100:00000200:1.0:1773941578.683424:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291492608, portal 25 00000100:00000001:1.0:1773941578.683425:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941578.683426:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00000100:00000040:1.0:1773941578.683430:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c0700 x1860112291492608/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941594 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941578.683436:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941578.683439:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941578.683440:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113deb28. 00000100:00000200:1.0:1773941578.683443:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291492608, offset 0 00000400:00000200:1.0:1773941578.683448:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941578.683456:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941578.683464:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941578.683470:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941578.683473:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941578.683477:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941578.683483:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941578.683485:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[126, 125] 00000400:00000200:1.0:1773941578.683488:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:1.0:1773941578.683493:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [237:237:256:4294967295] 192.168.204.124@tcp LPNI seq info [127:127:8:4294967295] 00000400:00000200:1.0:1773941578.683496:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941578.683506:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:1.0:1773941578.683511:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:1.0:1773941578.683514:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfd100. 00000800:00000200:1.0:1773941578.683521:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:1.0:1773941578.683529:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:1.0:1773941578.683533:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfd100 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941578.683550:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941578.683554:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941578.683557:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294970279 00000100:00000010:1.0:1773941578.683561:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3a80. 00000100:00000001:1.0:1773941578.683568:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1773941578.683575:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294970279 00000100:00000010:1.0:1773941578.683579:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3840. 00000100:00000001:1.0:1773941578.683582:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1773941578.683585:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941578.683592:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941578.683594:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941578.683596:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941578.683598:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 11 seconds 00000100:00000001:1.0:1773941578.683601:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941578.683604:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941578.683717:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941578.683728:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfd100. 00000400:00000200:0.0:1773941578.683735:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941578.683744:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941578.683751:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113deb28 00000400:00000010:0.0:1773941578.683755:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113deb28. 00000100:00000001:0.0:1773941578.683759:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941578.683768:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c0700 x1860112291492608/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941594 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941578.683780:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941578.683784:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d203c0700 x1860112291492608/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941594 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941578.683791:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941578.683794:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941578.685371:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941578.685385:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941578.685388:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941578.685390:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941578.685397:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941578.685403:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941578.685410:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941578.685429:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941578.685434:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941578.685439:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324607300 00000400:00000200:0.0:1773941578.685444:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941578.685450:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0x1009 [1] + 224 00000800:00000001:0.0:1773941578.685458:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941578.685471:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941578.685474:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941578.685478:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941578.685482:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941578.685487:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c0700 x1860112291492608/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941594 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941578.685498:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d203c0700 x1860112291492608/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941594 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941578.685520:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941578.685527:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941578.685531:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941578.685585:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941578.685592:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941578.685594:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941578.685597:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:1.0:1773941578.685601:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:1.0:1773941578.685603:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:1.0:1773941578.685613:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941578.685618:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c0700 x1860112291492608/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941594 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941578.685627:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c0700 x1860112291492608/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941594 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941578.685633:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941578.685634:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941578.685636:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941578.685639:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941578.685641:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941578.685643:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941578.685645:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941578.685651:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=79 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941578.685653:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941578.685655:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941578.685659:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:1.0:1773941578.685661:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941578.685664:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941578.685666:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941578.685670:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941578.685673:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941578.685676:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941578.685678:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941578.685680:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941578.685681:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941578.685684:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d203c0700 x1860112291492608/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941594 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941578.685689:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941578.685690:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941578.685691:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941578.685694:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941578.685696:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941578.685718:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941578.685720:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941578.685722:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941578.685723:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941578.685725:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941578.685726:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941578.685730:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941578.685732:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941578.685736:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941578.685739:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 299 00000400:00000200:1.0:1773941578.685745:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941578.685749:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941578.685752:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941578.685755:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941578.685757:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941578.685759:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941578.685762:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941578.685763:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941578.685765:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941578.685766:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941578.685768:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941578.685771:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941578.685774:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941578.685777:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941578.685779:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941578.685781:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941578.685785:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941578.685789:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941578.685793:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941578.685796:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941578.685801:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941578.685804:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941578.685814:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941578.685817:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941578.685822:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c0e00. 00000020:00000040:1.0:1773941578.685825:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941578.685828:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941578.685830:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941578.685832:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941578.685834:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941578.685836:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941578.685841:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d11188c00. 00000100:00000001:1.0:1773941578.685845:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941578.685852:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c0e00 x1860112291492736/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941578.685859:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941578.685862:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941578.685867:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001eb38943] to pc [ptlrpcd_rcv+-1] req@ffff988d203c0e00 x1860112291492736/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941578.685891:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941578.685894:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941578.685896:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941578.685898:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941578.685901:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941578.685902:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941578.685905:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941578.685908:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941578.685910:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941578.685916:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c0700 x1860112291492608/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941594 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941578.685921:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941578.685926:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941578.685927:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941578.685929:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941578.685934:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941578.685939:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c0e00 x1860112291492736/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941578.685939:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941578.685942:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941578.685951:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:1.0:1773941578.685953:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c0700 x1860112291492608/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941594 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941578.685954:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941578.685957:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941578.685959:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941578.685979:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941578.685982:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291492736:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941578.685986:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941578.685988:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941578.685990:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941578.685993:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941578.685998:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03522400. 02000000:00000001:3.0:1773941578.686002:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941578.686009:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00100000:1.0:1773941578.686009:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291492608:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000400:00000010:3.0:1773941578.686013:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00000100:00000001:1.0:1773941578.686018:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000400:00000010:3.0:1773941578.686020:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00000100:00000001:1.0:1773941578.686025:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000200:3.0:1773941578.686026:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291492736, portal 25 00000100:00000001:1.0:1773941578.686030:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941578.686034:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1773941578.686038:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000040:1.0:1773941578.686039:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c0700 x1860112291492608/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941594 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941578.686041:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631458304 : -113743078093312 : ffff988d203c0e00) 00000100:00000040:3.0:1773941578.686052:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c0e00 x1860112291492736/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941588 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941578.686059:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941578.686065:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1a1ab9c0. 00000100:00000001:3.0:1773941578.686066:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941578.686071:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000100:00000001:1.0:1773941578.686072:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941578.686076:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:1.0:1773941578.686081:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000400:00000010:3.0:1773941578.686082:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ace58. 00010000:00000001:1.0:1773941578.686087:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000200:3.0:1773941578.686088:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291492736, offset 0 00000020:00000001:1.0:1773941578.686091:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:3.0:1773941578.686095:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000020:00000040:1.0:1773941578.686096:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488800 refcount=4 00000020:00000001:1.0:1773941578.686101:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00000400:00000200:3.0:1773941578.686102:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941578.686107:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00000400:00000200:3.0:1773941578.686108:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941578.686114:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941578.686117:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00010000:1.0:1773941578.686117:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a40 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941578.686121:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941578.686123:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941578.686125:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[126, 127] 00000400:00000200:3.0:1773941578.686128:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941578.686130:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [238:238:256:4294967295] 192.168.204.114@tcp LPNI seq info [128:128:8:4294967295] 00000400:00000200:3.0:1773941578.686136:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941578.686144:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:3.0:1773941578.686153:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00010000:00000001:1.0:1773941578.686155:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000800:00000010:3.0:1773941578.686157:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74400. 00000800:00000200:3.0:1773941578.686165:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00010000:00010000:1.0:1773941578.686166:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a40 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941578.686173:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:3.0:1773941578.686178:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74400 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941578.686184:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941578.686189:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a40 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941578.686193:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941578.686200:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:3.0:1773941578.686203:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294970282 00010000:00000001:1.0:1773941578.686206:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941578.686210:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294970279 00010000:00000001:1.0:1773941578.686218:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:3.0:1773941578.686220:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294970282 00010000:00010000:1.0:1773941578.686224:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a40 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941578.686229:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294970279 00000100:00000001:3.0:1773941578.686235:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941578.686238:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:3.0:1773941578.686239:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941578.686244:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941578.686245:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941578.686248:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:1.0:1773941578.686251:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000100:00000001:3.0:1773941578.686256:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00010000:00010000:1.0:1773941578.686256:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a40 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941578.686264:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1773941578.686268:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00010000:1.0:1773941578.686269:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d01488800 already unused, calling callback (ffffffffc12b4c20) 00000100:00000001:3.0:1773941578.686270:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941578.686274:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941578.686276:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000001:3.0:1773941578.686277:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:00010000:1.0:1773941578.686282:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a40 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941578.686294:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000800:00000200:0.0:1773941578.686295:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00000001:1.0:1773941578.686299:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000800:00000010:0.0:1773941578.686302:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74400. 00000020:00000001:1.0:1773941578.686304:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941578.686309:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488800 refcount=5 00000400:00000200:0.0:1773941578.686312:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:1.0:1773941578.686314:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941578.686322:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00000400:00000200:0.0:1773941578.686324:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00010000:00000001:1.0:1773941578.686332:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:0.0:1773941578.686334:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ace58 00010000:00010000:1.0:1773941578.686339:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a40 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:0.0:1773941578.686339:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ace58. 00000100:00000001:0.0:1773941578.686347:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941578.686357:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c0e00 x1860112291492736/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941588 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000001:1.0:1773941578.686359:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941578.686365:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a40 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941578.686382:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 10000000:01000000:1.0:1773941578.686388:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000100:00000040:0.0:1773941578.686390:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c0e00 x1860112291492736/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941588 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000040:1.0:1773941578.686397:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941578.686401:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941578.686411:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000100:00000001:0.0:1773941578.686416:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941578.686417:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000100:00000001:0.0:1773941578.686421:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:1.0:1773941578.686430:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941578.686435:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941578.686438:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941578.686441:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941578.686444:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d01488800 with handle 0x8e86767622e2a40 from hash 00010000:00000001:1.0:1773941578.686449:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941578.686451:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941578.686454:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941578.686455:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941578.686457:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941578.686460:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941578.686463:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941578.686464:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941578.686467:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941578.686470:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941578.686474:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a40 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941578.686481:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941578.686482:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941578.686484:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941578.686486:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941578.686487:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941578.686488:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941578.686490:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941578.686493:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a40 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:1.0:1773941578.686502:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d04dd4e00 count: 0 00010000:00000001:1.0:1773941578.686515:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941578.686517:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941578.686521:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941578.686528:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c0700 x1860112291492608/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941594 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941578.686537:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941578.686540:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941578.686549:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d11189400. 02000000:00000001:1.0:1773941578.686553:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941578.686555:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941578.686557:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941578.686560:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941578.686563:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d181fc800. 02000000:00000001:1.0:1773941578.686566:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941578.686568:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941578.686570:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c0700. 00000100:00000001:1.0:1773941578.686573:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941578.686575:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941578.686577:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941578.686579:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941578.686582:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941578.686586:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d04d678c0. 00000020:00000010:1.0:1773941578.686591:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d181fc800. 00000020:00000010:1.0:1773941578.686597:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d181fce00. 00000040:00000010:1.0:1773941578.686602:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d181fd000. 00000020:00000010:1.0:1773941578.686609:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941578.686614:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d181fc400. 80000000:00000010:1.0:1773941578.686618:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d054f3720. 00040000:00000010:1.0:1773941578.686624:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d181fd600. 20000000:00000010:1.0:1773941578.686630:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d181fd800. 10000000:00000040:1.0:1773941578.686635:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:1.0:1773941578.686638:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941578.686643:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d181fce00. 00000040:00000010:1.0:1773941578.686648:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d181fd000. 00000020:00000010:1.0:1773941578.686652:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941578.686656:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d181fc400. 80000000:00000010:1.0:1773941578.686660:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d054f3720. 00040000:00000010:1.0:1773941578.686665:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d181fd600. 20000000:00000010:1.0:1773941578.686669:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d181fd800. 00000020:00000010:1.0:1773941578.686672:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d181fc800. 10000000:00000010:1.0:1773941578.686675:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d04d678c0. 10000000:01000000:1.0:1773941578.686678:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941578.686689:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941578.686691:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941578.686694:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941578.686696:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941578.686698:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941578.686700:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941578.687697:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941578.687717:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941578.687720:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941578.687723:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941578.687728:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941578.687734:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941578.687740:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941578.687744:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941578.687747:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941578.687751:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324607380 00000400:00000200:0.0:1773941578.687755:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941578.687761:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0x1019 [1] + 0 00000800:00000001:0.0:1773941578.687767:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941578.687776:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941578.687780:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941578.687784:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941578.687788:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941578.687793:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c0e00 x1860112291492736/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941588 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941578.687804:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c0e00 x1860112291492736/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941588 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941578.687825:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941578.687830:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941578.687835:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941578.687854:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941578.687858:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941578.687861:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941578.687862:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941578.687865:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941578.687870:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941578.687871:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941578.687874:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941578.687879:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c0e00 x1860112291492736/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941588 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941578.687888:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c0e00 x1860112291492736/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941588 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941578.687893:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941578.687894:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941578.687896:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941578.687897:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941578.687898:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941578.687899:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941578.687901:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941578.687903:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=79 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941578.687906:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941578.687908:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941578.687909:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941578.687911:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941578.687913:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941578.687915:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941578.687918:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c0e00 x1860112291492736/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941588 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941578.687922:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941578.687924:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941578.687931:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941578.687933:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941578.687938:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941578.687940:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941578.687941:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941578.687942:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941578.687944:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941578.687946:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941578.687947:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941578.687950:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941578 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941578.687952:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941578.687958:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941578.687990:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941578.687993:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941578.687995:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941578.687997:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941578.688000:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941578.688003:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941578.688005:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941578.688009:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941578.688013:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941578.688015:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941578.688245:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941578.688252:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941578.688258:0:14785:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000040:2.0:1773941578.688264:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c0e00 x1860112291492736/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941588 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1773941578.688268:0:14785:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:3.0:1773941578.688274:0:14785:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00100000:2.0:1773941578.688286:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291492736:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941578.688299:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941578.688303:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1773941578.688307:0:14785:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000040:2.0:1773941578.688309:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c0e00 x1860112291492736/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941588 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:01000000:3.0:1773941578.688310:0:14785:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:3.0:1773941578.688314:0:14785:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941578.688318:0:14785:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941578.688322:0:14785:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941578.688326:0:14785:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:2.0:1773941578.688326:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:3.0:1773941578.688330:0:14785:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941578.688332:0:14785:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941578.688332:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 10000000:00000001:3.0:1773941578.688337:0:14785:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941578.688338:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03522400. 02000000:00000001:2.0:1773941578.688344:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1773941578.688346:0:14785:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000001:2.0:1773941578.688348:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000100:00080000:3.0:1773941578.688351:0:14785:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000020:00000040:2.0:1773941578.688352:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941578.688356:0:14785:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000020:00000001:2.0:1773941578.688358:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941578.688359:0:14785:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941578.688363:0:14785:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941578.688364:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d11188c00. 00000100:00080000:3.0:1773941578.688366:0:14785:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:3.0:1773941578.688370:0:14785:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:2.0:1773941578.688371:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 10000000:01000000:3.0:1773941578.688372:0:14785:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 02000000:00000001:2.0:1773941578.688375:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 10000000:00000040:3.0:1773941578.688376:0:14785:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00000010:2.0:1773941578.688380:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c0e00. 00000100:00000001:2.0:1773941578.688388:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00080000:3.0:1773941578.688392:0:14785:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:3.0:1773941578.688394:0:14785:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941578.688396:0:14785:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:3.0:1773941578.688401:0:14785:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:2.0:1773941578.688401:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941578.688407:0:14785:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941578.688407:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1773941578.688410:0:14785:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941578.688414:0:14785:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941578.688416:0:14785:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941578.688418:0:14785:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941578.688418:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941578.688423:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941578.688428:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941578.688430:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941578.691997:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d04dd4e00. 00010000:00000010:1.1:1773941578.692005:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d01488800. 00000020:00000001:2.0:1773941578.747144:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941578.747152:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941578.747157:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1773941579.771179:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941579.771188:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941579.771193:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941579.963024:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294970282, at 4294970304 00000100:00000040:0.0:1773941579.963024:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294970282, at 4294970304 00000100:00000010:1.0:1773941579.963039:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3840. 00000100:00000010:0.0:1773941579.963041:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3a80. 10000000:00000001:1.0:1773941580.347058:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941580.347070:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941580.347076:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941580.347079:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941580.347081:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941580.347082:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941580.347086:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941580.347088:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941580.347092:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:1.0:1773941580.347095:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941580.347098:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941580.347111:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c0700. 00000020:00000040:1.0:1773941580.347117:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941580.347122:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941580.347126:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941580.347132:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941580.347141:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d181fd800. 00000100:00000001:1.0:1773941580.347149:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941580.347154:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941580.347157:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941580.347166:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d04dd4e00. 00010000:00000001:1.0:1773941580.347171:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941580.347174:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d01488800. 00000020:00000001:1.0:1773941580.347178:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941580.347181:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d01488800 with handle 0x8e86767622e2a47 to hash 00000020:00000001:1.0:1773941580.347183:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941580.347184:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941580.347187:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941580.347210:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941580.347213:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941580.347217:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a47 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941580.347227:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a47 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941580.347238:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a47 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941580.347243:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941580.347245:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941580.347251:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941580.347253:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330528610752 : -113743180940864 : ffff988d1a1ab9c0) 00000100:00000001:1.0:1773941580.347255:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941580.347257:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00000100:00000001:1.0:1773941580.347261:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941580.347266:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941580.347268:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941580.347270:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941580.347281:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c0700 x1860112291492864/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941580.347288:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941580.347290:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941580.347293:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941580.347296:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941580.347299:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941580.347303:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291492864:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941580.347308:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941580.347316:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941580.347319:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941580.347322:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941580.347334:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d11189400. 02000000:00000001:1.0:1773941580.347337:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941580.347347:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941580.347352:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c5d8. 00000400:00000010:1.0:1773941580.347361:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000100:00000200:1.0:1773941580.347370:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291492864, portal 25 00000100:00000001:1.0:1773941580.347372:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941580.347373:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00000100:00000040:1.0:1773941580.347378:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c0700 x1860112291492864/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941596 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941580.347385:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941580.347387:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941580.347390:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113def68. 00000100:00000200:1.0:1773941580.347392:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291492864, offset 0 00000400:00000200:1.0:1773941580.347399:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941580.347408:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941580.347416:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941580.347421:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941580.347425:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941580.347429:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941580.347431:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941580.347434:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[128, 127] 00000400:00000200:1.0:1773941580.347436:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:1.0:1773941580.347440:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [239:239:256:4294967295] 192.168.204.124@tcp LPNI seq info [129:129:8:4294967295] 00000400:00000200:1.0:1773941580.347443:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941580.347451:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:1.0:1773941580.347456:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:1.0:1773941580.347460:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfd700. 00000800:00000200:1.0:1773941580.347467:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:1.0:1773941580.347475:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:1.0:1773941580.347479:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfd700 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941580.347501:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941580.347505:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941580.347507:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294971943 00000100:00000010:1.0:1773941580.347512:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3840. 00000100:00000001:1.0:1773941580.347526:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1773941580.347534:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294971943 00000100:00000010:1.0:1773941580.347537:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3720. 00000100:00000001:1.0:1773941580.347539:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1773941580.347542:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941580.347544:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941580.347546:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941580.347547:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941580.347550:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 11 seconds 00000100:00000001:1.0:1773941580.347553:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941580.347556:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941580.347737:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941580.347745:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfd700. 00000400:00000200:0.0:1773941580.347751:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941580.347759:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941580.347766:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113def68 00000400:00000010:0.0:1773941580.347768:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113def68. 00000100:00000001:0.0:1773941580.347785:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941580.347792:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c0700 x1860112291492864/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941596 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941580.347802:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941580.347805:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d203c0700 x1860112291492864/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941596 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941580.347810:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941580.347812:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941580.348819:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941580.348832:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941580.348834:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941580.348836:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941580.348842:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941580.348846:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941580.348852:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941580.348856:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941580.348862:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941580.348867:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324607400 00000400:00000200:0.0:1773941580.348871:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941580.348879:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0x1021 [1] + 224 00000800:00000001:0.0:1773941580.348887:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941580.348896:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941580.348898:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941580.348902:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941580.348907:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941580.348913:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c0700 x1860112291492864/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941596 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941580.348925:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d203c0700 x1860112291492864/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941596 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941580.348948:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941580.348956:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000100:00000001:1.0:1773941580.348983:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941580.348990:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941580.348992:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941580.348995:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:1.0:1773941580.349000:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:1.0:1773941580.349001:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:1.0:1773941580.349004:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941580.349009:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c0700 x1860112291492864/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941596 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941580.349022:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c0700 x1860112291492864/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941596 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000800:00000001:0.0:1773941580.349025:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941580.349035:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941580.349038:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941580.349041:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941580.349043:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941580.349045:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941580.349048:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941580.349050:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941580.349056:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=81 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941580.349066:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941580.349069:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941580.349072:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:1.0:1773941580.349075:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941580.349078:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941580.349080:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941580.349085:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941580.349088:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941580.349090:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941580.349092:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941580.349094:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941580.349096:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941580.349098:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d203c0700 x1860112291492864/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941596 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941580.349104:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941580.349105:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941580.349107:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941580.349109:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941580.349112:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941580.349113:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941580.349114:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941580.349115:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941580.349116:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941580.349117:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941580.349118:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941580.349120:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941580.349121:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941580.349124:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941580.349126:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 301 00000400:00000200:1.0:1773941580.349130:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941580.349134:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941580.349137:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941580.349139:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941580.349144:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941580.349146:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941580.349149:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941580.349150:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941580.349152:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941580.349154:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941580.349156:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941580.349157:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941580.349159:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941580.349160:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941580.349162:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941580.349163:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941580.349165:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941580.349168:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941580.349170:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941580.349173:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941580.349175:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941580.349177:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941580.349179:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941580.349180:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941580.349185:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c0e00. 00000020:00000040:1.0:1773941580.349187:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941580.349190:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941580.349191:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941580.349193:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941580.349194:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941580.349196:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941580.349199:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d1118a800. 00000100:00000001:1.0:1773941580.349202:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941580.349207:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c0e00 x1860112291492992/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941580.349213:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941580.349218:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941580.349223:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001eb38943] to pc [ptlrpcd_rcv+-1] req@ffff988d203c0e00 x1860112291492992/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941580.349242:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941580.349244:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941580.349246:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941580.349248:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941580.349250:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941580.349251:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941580.349252:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941580.349254:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941580.349255:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941580.349259:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c0700 x1860112291492864/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941596 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941580.349264:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941580.349266:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941580.349268:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941580.349270:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941580.349274:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c0700 x1860112291492864/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941596 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941580.349280:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1773941580.349281:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291492864:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941580.349286:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941580.349288:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941580.349290:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941580.349291:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941580.349294:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941580.349296:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941580.349300:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c0700 x1860112291492864/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941596 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941580.349305:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941580.349307:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1a1ab9c0. 00000100:00000040:2.0:1773941580.349309:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c0e00 x1860112291492992/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941580.349310:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941580.349315:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:1.0:1773941580.349321:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941580.349325:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941580.349328:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1773941580.349331:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000040:1.0:1773941580.349332:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488800 refcount=4 00000100:00000001:2.0:1773941580.349337:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941580.349337:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941580.349343:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 02000000:00000001:2.0:1773941580.349345:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00010000:00010000:1.0:1773941580.349347:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a47 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941580.349349:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941580.349356:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941580.349358:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00100000:2.0:1773941580.349363:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291492992:192.168.204.114@tcp:250:ll_cfg_requeue.0 00010000:00010000:1.0:1773941580.349364:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a47 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941580.349371:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941580.349375:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00000001:1.0:1773941580.349376:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 02000000:00000001:2.0:1773941580.349378:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941580.349380:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a47 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941580.349381:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:1.0:1773941580.349388:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 02000000:00000010:2.0:1773941580.349389:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a4800. 00010000:00000001:1.0:1773941580.349391:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 02000000:00000001:2.0:1773941580.349394:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941580.349395:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a47 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941580.349401:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941580.349407:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000010:2.0:1773941580.349408:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00010000:00000001:1.0:1773941580.349410:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:2.0:1773941580.349413:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00010000:00000001:1.0:1773941580.349415:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000100:00000200:2.0:1773941580.349419:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291492992, portal 25 00010000:00010000:1.0:1773941580.349420:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a47 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941580.349425:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941580.349427:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631458304 : -113743078093312 : ffff988d203c0e00) 00010000:00010000:1.0:1773941580.349430:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d01488800 already unused, calling callback (ffffffffc12b4c20) 10000000:00000001:1.0:1773941580.349432:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941580.349434:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a47 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941580.349442:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c0e00 x1860112291492992/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941590 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941580.349454:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000100:00000001:2.0:1773941580.349456:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:1.0:1773941580.349457:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941580.349460:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000040:2.0:1773941580.349461:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000020:00000040:1.0:1773941580.349463:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488800 refcount=5 00000020:00000001:1.0:1773941580.349466:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00000400:00000010:2.0:1773941580.349467:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516b28. 00010000:00000001:1.0:1773941580.349468:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00000100:00000200:2.0:1773941580.349473:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291492992, offset 0 00010000:00000001:1.0:1773941580.349474:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941580.349477:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a47 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941580.349483:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 10000000:00000001:1.0:1773941580.349489:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:2.0:1773941580.349491:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 10000000:00010000:1.0:1773941580.349493:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a47 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941580.349498:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 10000000:01000000:1.0:1773941580.349503:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:2.0:1773941580.349504:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 10000000:00000040:1.0:1773941580.349508:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000400:00000200:2.0:1773941580.349510:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 10000000:00000001:1.0:1773941580.349511:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941580.349515:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00010000:1.0:1773941580.349516:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000400:00000200:2.0:1773941580.349519:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941580.349522:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[128, 129] 00000400:00000200:2.0:1773941580.349524:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:2.0:1773941580.349527:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [240:240:256:4294967295] 192.168.204.114@tcp LPNI seq info [130:130:8:4294967295] 00010000:00000001:1.0:1773941580.349528:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:2.0:1773941580.349531:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941580.349534:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941580.349537:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941580.349540:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:2.0:1773941580.349541:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00010000:00000001:1.0:1773941580.349543:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941580.349548:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000020:00000040:1.0:1773941580.349548:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d01488800 with handle 0x8e86767622e2a47 from hash 00010000:00000001:1.0:1773941580.349551:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000800:00000010:2.0:1773941580.349554:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0dbd00. 00010000:00000001:1.0:1773941580.349554:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941580.349557:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941580.349559:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941580.349561:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000800:00000200:2.0:1773941580.349564:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00010000:00000001:1.0:1773941580.349564:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941580.349567:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941580.349570:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941580.349571:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941580.349574:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:2.0:1773941580.349578:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0dbd00 type 1, nob 616 niov 1 nkiov 1 10000000:00000001:1.0:1773941580.349579:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941580.349585:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a47 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941580.349590:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941580.349591:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941580.349592:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941580.349593:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000100:00000001:2.0:1773941580.349594:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941580.349595:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941580.349598:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941580.349601:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:2.0:1773941580.349602:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:1.0:1773941580.349604:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a47 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941580.349605:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294971945 00010000:00000040:1.0:1773941580.349615:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d04dd4e00 count: 0 00000100:00000001:2.0:1773941580.349616:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294971943 00000100:00000001:2.0:1773941580.349622:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294971945 00000100:00000001:2.0:1773941580.349626:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294971943 00010000:00000001:1.0:1773941580.349631:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941580.349633:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941580.349639:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941580.349640:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941580.349644:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941580.349645:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c0700 x1860112291492864/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941596 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941580.349649:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941580.349655:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941580.349659:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941580.349664:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1773941580.349666:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 02000000:00000010:1.0:1773941580.349667:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d11189400. 00000100:00000001:2.0:1773941580.349682:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 02000000:00000001:1.0:1773941580.349685:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1773941580.349687:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1773941580.349689:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:2.0:1773941580.349691:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000020:00000040:1.0:1773941580.349692:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941580.349697:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941580.349700:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941580.349702:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941580.349707:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d181fd800. 02000000:00000001:1.0:1773941580.349714:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941580.349716:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941580.349718:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c0700. 00000100:00000001:1.0:1773941580.349722:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941580.349723:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941580.349725:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941580.349727:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941580.349731:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000800:00000200:0.0:1773941580.349732:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941580.349737:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0dbd00. 10000000:00000010:1.0:1773941580.349740:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d04d67400. 00000400:00000200:0.0:1773941580.349744:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000010:1.0:1773941580.349745:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d181fd800. 00000020:00000010:1.0:1773941580.349759:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d181fd600. 00000400:00000200:0.0:1773941580.349765:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000040:00000010:1.0:1773941580.349768:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d181fc400. 00000400:00000200:0.0:1773941580.349771:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516b28 00000400:00000010:0.0:1773941580.349775:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516b28. 00000020:00000010:1.0:1773941580.349776:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941580.349788:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d181fd000. 00000100:00000001:0.0:1773941580.349788:0:2393:0:(events.c:34:request_out_callback()) Process entered 80000000:00000010:1.0:1773941580.349799:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d054f37e0. 00000100:00000200:0.0:1773941580.349804:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c0e00 x1860112291492992/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941590 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00040000:00000010:1.0:1773941580.349807:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d181fce00. 20000000:00000010:1.0:1773941580.349819:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d181fc200. 00000100:00000001:0.0:1773941580.349821:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 10000000:00000040:1.0:1773941580.349825:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 00000100:00000040:0.0:1773941580.349827:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c0e00 x1860112291492992/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941590 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000001:1.0:1773941580.349832:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941580.349846:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d181fd600. 00000100:00000001:0.0:1773941580.349848:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000040:00000010:1.0:1773941580.349852:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d181fc400. 00000100:00000001:0.0:1773941580.349852:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:1.0:1773941580.349860:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941580.349865:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d181fd000. 80000000:00000010:1.0:1773941580.349867:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d054f37e0. 00040000:00000010:1.0:1773941580.349873:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d181fce00. 20000000:00000010:1.0:1773941580.349877:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d181fc200. 00000020:00000010:1.0:1773941580.349881:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d181fd800. 10000000:00000010:1.0:1773941580.349883:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d04d67400. 10000000:01000000:1.0:1773941580.349885:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941580.349888:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941580.349889:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941580.349891:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941580.349892:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941580.349893:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941580.349895:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941580.351002:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941580.351011:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941580.351013:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941580.351014:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941580.351019:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941580.351022:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941580.351027:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941580.351029:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941580.351033:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941580.351035:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324607480 00000400:00000200:0.0:1773941580.351038:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941580.351042:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0x1031 [1] + 0 00000800:00000001:0.0:1773941580.351046:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941580.351061:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941580.351063:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941580.351067:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941580.351071:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941580.351076:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c0e00 x1860112291492992/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941590 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941580.351097:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c0e00 x1860112291492992/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941590 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941580.351117:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941580.351124:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941580.351128:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941580.351149:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941580.351152:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941580.351155:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941580.351156:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941580.351159:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941580.351162:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941580.351163:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941580.351165:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941580.351170:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c0e00 x1860112291492992/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941590 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941580.351178:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c0e00 x1860112291492992/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941590 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941580.351183:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941580.351184:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941580.351185:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941580.351187:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941580.351188:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941580.351189:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941580.351190:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941580.351193:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=81 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941580.351195:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941580.351197:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941580.351198:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941580.351201:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941580.351202:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941580.351204:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941580.351213:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c0e00 x1860112291492992/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941590 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941580.351218:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941580.351220:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941580.351222:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941580.351224:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941580.351228:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941580.351229:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941580.351230:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941580.351231:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941580.351233:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941580.351235:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941580.351236:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:2.0:1773941580.351240:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941580 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941580.351242:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941580.351248:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:2.0:1773941580.351249:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941580.351251:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941580.351252:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941580.351253:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941580.351256:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:2.0:1773941580.351258:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941580.351260:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941580.351263:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941580.351265:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:2.0:1773941580.351267:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941580.351535:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941580.351540:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941580.351557:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c0e00 x1860112291492992/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941590 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941580.351562:0:14786:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:3.0:1773941580.351571:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291492992:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00080000:2.0:1773941580.351576:0:14786:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00000001:3.0:1773941580.351579:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1773941580.351581:0:14786:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941580.351586:0:14786:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:3.0:1773941580.351589:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 10000000:01000000:2.0:1773941580.351589:0:14786:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:2.0:1773941580.351595:0:14786:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000040:3.0:1773941580.351597:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c0e00 x1860112291492992/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941590 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000001:00000001:2.0:1773941580.351610:0:14786:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:3.0:1773941580.351611:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:2.0:1773941580.351615:0:14786:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941580.351616:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:2.0:1773941580.351621:0:14786:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000010:3.0:1773941580.351623:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a4800. 00000001:00000001:2.0:1773941580.351624:0:14786:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:3.0:1773941580.351632:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:2.0:1773941580.351632:0:14786:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941580.351640:0:14786:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941580.351641:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941580.351650:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941580.351650:0:14786:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941580.351659:0:14786:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941580.351669:0:14786:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000020:00000001:3.0:1773941580.351683:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941580.351687:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d1118a800. 00000100:00000001:2.0:1773941580.351687:0:14786:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:2.0:1773941580.351693:0:14786:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941580.351695:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00080000:2.0:1773941580.351697:0:14786:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 02000000:00000001:3.0:1773941580.351698:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1773941580.351706:0:14786:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000010:3.0:1773941580.351708:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c0e00. 10000000:01000000:2.0:1773941580.351710:0:14786:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:2.0:1773941580.351713:0:14786:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00000001:3.0:1773941580.351721:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941580.351725:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941580.351729:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1773941580.351730:0:14786:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:2.0:1773941580.351733:0:14786:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941580.351740:0:14786:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:2.0:1773941580.351744:0:14786:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:3.0:1773941580.351747:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941580.351748:0:14786:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941580.351754:0:14786:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:3.0:1773941580.351755:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941580.351760:0:14786:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941580.351764:0:14786:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941580.351766:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941580.351768:0:14786:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941580.351775:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941580.355002:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d04dd4e00. 00010000:00000010:1.1:1773941580.355011:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d01488800. 00000020:00000001:2.0:1773941580.795108:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941580.795115:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941580.795120:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941581.371086:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941581.371095:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941581.371099:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941581.371102:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941581.371103:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941581.371105:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941581.371108:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941581.371133:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941581.371135:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:1.0:1773941581.371138:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941581.371139:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941581.371147:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c0700. 00000020:00000040:1.0:1773941581.371152:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941581.371157:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941581.371160:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941581.371166:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941581.371173:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d181fc200. 00000100:00000001:1.0:1773941581.371181:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941581.371185:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941581.371189:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941581.371196:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d04dd4e00. 00010000:00000001:1.0:1773941581.371202:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941581.371205:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d01488800. 00000020:00000001:1.0:1773941581.371210:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941581.371212:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d01488800 with handle 0x8e86767622e2a4e to hash 00000020:00000001:1.0:1773941581.371214:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941581.371215:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941581.371218:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941581.371221:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941581.371224:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941581.371227:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a4e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941581.371237:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a4e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941581.371248:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a4e lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941581.371253:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941581.371255:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941581.371264:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941581.371266:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330528610752 : -113743180940864 : ffff988d1a1ab9c0) 00000100:00000001:1.0:1773941581.371269:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941581.371270:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00000100:00000001:1.0:1773941581.371275:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941581.371279:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941581.371282:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941581.371283:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941581.371294:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c0700 x1860112291493120/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941581.371311:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941581.371313:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941581.371318:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941581.371320:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941581.371322:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941581.371325:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291493120:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941581.371329:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941581.371337:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941581.371340:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941581.371343:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941581.371349:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d11189400. 02000000:00000001:1.0:1773941581.371351:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941581.371362:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941581.371368:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c5d8. 00000400:00000010:1.0:1773941581.371376:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000100:00000200:1.0:1773941581.371386:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291493120, portal 25 00000100:00000001:1.0:1773941581.371388:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941581.371389:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00000100:00000040:1.0:1773941581.371395:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c0700 x1860112291493120/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941597 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941581.371401:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941581.371403:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941581.371406:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de220. 00000100:00000200:1.0:1773941581.371413:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291493120, offset 0 00000400:00000200:1.0:1773941581.371419:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941581.371427:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941581.371436:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941581.371440:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941581.371444:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941581.371448:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941581.371450:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941581.371453:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[130, 129] 00000400:00000200:1.0:1773941581.371456:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:1.0:1773941581.371460:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [241:241:256:4294967295] 192.168.204.124@tcp LPNI seq info [131:131:8:4294967295] 00000400:00000200:1.0:1773941581.371463:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941581.371472:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:1.0:1773941581.371476:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:1.0:1773941581.371481:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfdc00. 00000800:00000200:1.0:1773941581.371487:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:1.0:1773941581.371495:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:1.0:1773941581.371499:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfdc00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941581.371517:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941581.371520:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941581.371523:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294972967 00000100:00000001:1.0:1773941581.371531:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294971945 00000100:00000001:1.0:1773941581.371536:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294972967 00000100:00000001:1.0:1773941581.371540:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294971945 00000100:00000001:1.0:1773941581.371542:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941581.371545:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941581.371547:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941581.371549:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941581.371551:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 11 seconds 00000100:00000001:1.0:1773941581.371554:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941581.371563:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941581.371692:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941581.371700:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfdc00. 00000400:00000200:0.0:1773941581.371706:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941581.371714:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941581.371720:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de220 00000400:00000010:0.0:1773941581.371722:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de220. 00000100:00000001:0.0:1773941581.371726:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941581.371733:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c0700 x1860112291493120/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941597 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941581.371742:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941581.371745:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d203c0700 x1860112291493120/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941597 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941581.371751:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941581.371752:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941581.372725:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941581.372738:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941581.372741:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941581.372742:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941581.372748:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941581.372752:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941581.372758:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941581.372761:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941581.372766:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941581.372769:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324607500 00000400:00000200:0.0:1773941581.372774:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941581.372778:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0x1039 [1] + 224 00000800:00000001:0.0:1773941581.372798:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941581.372806:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941581.372808:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941581.372811:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941581.372814:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941581.372818:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c0700 x1860112291493120/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941597 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941581.372826:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d203c0700 x1860112291493120/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941597 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941581.372841:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941581.372848:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941581.372850:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941581.372875:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941581.372882:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941581.372884:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941581.372887:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:1.0:1773941581.372891:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:1.0:1773941581.372893:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:1.0:1773941581.372896:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941581.372901:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c0700 x1860112291493120/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941597 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941581.372909:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c0700 x1860112291493120/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941597 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941581.372914:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941581.372915:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941581.372917:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941581.372920:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941581.372922:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941581.372924:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941581.372926:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941581.372931:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=82 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941581.372934:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941581.372937:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941581.372941:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:1.0:1773941581.372944:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941581.372948:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941581.372951:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941581.372956:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941581.372982:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941581.372987:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941581.372990:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941581.372992:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941581.372994:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941581.372999:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d203c0700 x1860112291493120/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941597 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941581.373015:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941581.373017:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941581.373018:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941581.373021:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941581.373023:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941581.373024:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941581.373026:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941581.373027:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941581.373027:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941581.373029:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941581.373030:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941581.373032:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941581.373034:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941581.373036:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941581.373038:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 303 00000400:00000200:1.0:1773941581.373042:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941581.373045:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941581.373049:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941581.373051:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941581.373053:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941581.373055:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941581.373057:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941581.373059:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941581.373061:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941581.373063:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941581.373065:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941581.373066:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941581.373068:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941581.373069:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941581.373071:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941581.373072:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941581.373075:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941581.373081:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941581.373083:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941581.373086:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941581.373089:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941581.373091:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941581.373093:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941581.373094:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941581.373097:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c1500. 00000020:00000040:1.0:1773941581.373099:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941581.373101:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941581.373102:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941581.373104:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941581.373106:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941581.373108:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941581.373111:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d1118a400. 00000100:00000001:1.0:1773941581.373114:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941581.373118:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c1500 x1860112291493248/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941581.373124:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941581.373126:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941581.373129:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000022b3efad] to pc [ptlrpcd_rcv+-1] req@ffff988d203c1500 x1860112291493248/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941581.373148:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941581.373149:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941581.373151:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941581.373153:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941581.373155:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941581.373156:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941581.373158:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941581.373159:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941581.373161:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941581.373164:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c0700 x1860112291493120/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941597 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941581.373178:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941581.373181:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941581.373183:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941581.373185:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941581.373189:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c0700 x1860112291493120/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941597 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941581.373193:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1773941581.373198:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291493120:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941581.373201:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941581.373204:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941581.373207:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941581.373207:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941581.373213:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941581.373215:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941581.373224:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c0700 x1860112291493120/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941597 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941581.373226:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c1500 x1860112291493248/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941581.373242:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941581.373248:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941581.373254:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941581.373256:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:1.0:1773941581.373263:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000001:3.0:1773941581.373269:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:1.0:1773941581.373270:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941581.373274:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:3.0:1773941581.373277:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941581.373277:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488800 refcount=4 00000020:00000001:1.0:1773941581.373285:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941581.373288:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 02000000:00000001:3.0:1773941581.373289:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00010000:00010000:1.0:1773941581.373291:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a4e lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941581.373295:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941581.373306:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 02000000:00000001:3.0:1773941581.373307:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941581.373318:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a4e lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1773941581.373322:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291493248:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941581.373344:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941581.373350:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00000001:1.0:1773941581.373350:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 02000000:00000001:3.0:1773941581.373355:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941581.373356:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a4e lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941581.373362:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941581.373372:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03523800. 00010000:00000001:1.0:1773941581.373372:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941581.373375:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941581.373379:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a4e lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941581.373380:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941581.373392:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:3.0:1773941581.373393:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941581.373396:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941581.373401:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000010:3.0:1773941581.373402:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00010000:00010000:1.0:1773941581.373404:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a4e lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941581.373413:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d01488800 already unused, calling callback (ffffffffc12b4c20) 00000400:00000010:3.0:1773941581.373414:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 10000000:00000001:1.0:1773941581.373415:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941581.373419:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a4e lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:3.0:1773941581.373424:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291493248, portal 25 00010000:00000001:1.0:1773941581.373426:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941581.373429:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941581.373433:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:3.0:1773941581.373435:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000020:00000040:1.0:1773941581.373436:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488800 refcount=5 00000020:00000001:1.0:1773941581.373439:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00000100:00000001:3.0:1773941581.373440:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631460096 : -113743078091520 : ffff988d203c1500) 00010000:00000001:1.0:1773941581.373442:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941581.373446:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941581.373448:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a4e lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:1.0:1773941581.373464:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000040:3.0:1773941581.373469:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c1500 x1860112291493248/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941591 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00010000:1.0:1773941581.373472:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a4e lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1773941581.373507:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941581.373518:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000100:00000001:3.0:1773941581.373520:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 10000000:00000001:1.0:1773941581.373520:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941581.373531:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941581.373534:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000100:00000040:3.0:1773941581.373538:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00000001:1.0:1773941581.373541:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941581.373544:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000400:00000010:3.0:1773941581.373546:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acaa0. 00010000:00000001:1.0:1773941581.373547:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941581.373552:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1773941581.373556:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291493248, offset 0 00000020:00000040:1.0:1773941581.373556:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d01488800 with handle 0x8e86767622e2a4e from hash 00010000:00000001:1.0:1773941581.373563:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941581.373567:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941581.373568:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941581.373569:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941581.373571:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941581.373572:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941581.373578:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:3.0:1773941581.373579:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00000001:1.0:1773941581.373581:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941581.373590:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941581.373593:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941581.373598:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:1.0:1773941581.373601:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a4e lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941581.373610:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941581.373621:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:1.0:1773941581.373623:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941581.373627:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:3.0:1773941581.373631:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00000001:1.0:1773941581.373633:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941581.373635:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941581.373637:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941581.373641:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:3.0:1773941581.373642:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941581.373643:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941581.373647:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a4e lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941581.373667:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000040:1.0:1773941581.373670:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d04dd4e00 count: 0 00000400:00000200:3.0:1773941581.373674:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[130, 131] 00000400:00000200:3.0:1773941581.373684:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00010000:00000001:1.0:1773941581.373690:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:3.0:1773941581.373692:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [242:242:256:4294967295] 192.168.204.114@tcp LPNI seq info [132:132:8:4294967295] 00010000:00000001:1.0:1773941581.373693:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941581.373702:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941581.373709:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c0700 x1860112291493120/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941597 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941581.373719:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941581.373730:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941581.373734:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941581.373737:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d11189400. 02000000:00000001:1.0:1773941581.373744:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941581.373748:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000400:00000200:3.0:1773941581.373753:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000020:00000040:1.0:1773941581.373753:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941581.373764:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941581.373769:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d181fc200. 00000800:00000200:3.0:1773941581.373778:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 02000000:00000001:1.0:1773941581.373778:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941581.373783:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941581.373787:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c0700. 00000100:00000001:1.0:1773941581.373791:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941581.373792:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:3.0:1773941581.373794:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74400. 10000000:00000001:1.0:1773941581.373795:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941581.373802:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941581.373806:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000800:00000200:3.0:1773941581.373812:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 10000000:00000010:1.0:1773941581.373812:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d04d67d40. 00000020:00000010:1.0:1773941581.373821:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d181fc200. 00000800:00000200:3.0:1773941581.373827:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000020:00000010:1.0:1773941581.373829:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d181fce00. 00000040:00000010:1.0:1773941581.373837:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d181fd000. 00000800:00000200:3.0:1773941581.373838:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74400 type 1, nob 616 niov 1 nkiov 1 00000020:00000010:1.0:1773941581.373843:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941581.373848:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d181fc400. 80000000:00000010:1.0:1773941581.373852:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d054f37e0. 00000100:00000001:3.0:1773941581.373857:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00040000:00000010:1.0:1773941581.373857:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d181fd600. 20000000:00000010:1.0:1773941581.373868:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d181fd800. 00000400:00000200:3.0:1773941581.373869:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 10000000:00000040:1.0:1773941581.373874:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 00000100:00000001:3.0:1773941581.373875:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294972969 10000000:00000001:1.0:1773941581.373879:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941581.373887:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d181fce00. 00000100:00000001:3.0:1773941581.373888:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294972967 00000040:00000010:1.0:1773941581.373891:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d181fd000. 00000100:00000001:3.0:1773941581.373899:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294972969 00000020:00000010:1.0:1773941581.373902:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941581.373910:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d181fc400. 00000100:00000001:3.0:1773941581.373911:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294972967 80000000:00000010:1.0:1773941581.373913:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d054f37e0. 00040000:00000010:1.0:1773941581.373919:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d181fd600. 00000800:00000200:0.0:1773941581.373921:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 20000000:00000010:1.0:1773941581.373922:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d181fd800. 00000800:00000010:0.0:1773941581.373933:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74400. 00000020:00000010:1.0:1773941581.373935:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d181fc200. 00000100:00000001:3.0:1773941581.373936:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:0.0:1773941581.373938:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 10000000:00000010:1.0:1773941581.373939:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d04d67d40. 00000100:00000001:3.0:1773941581.373944:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 10000000:01000000:1.0:1773941581.373949:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 00000400:00000200:0.0:1773941581.373950:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:3.0:1773941581.373952:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941581.374008:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941581.374014:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 00000400:00000200:0.0:1773941581.374015:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acaa0 10000000:00000001:1.0:1773941581.374018:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 00000400:00000010:0.0:1773941581.374019:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acaa0. 10000000:00000001:1.0:1773941581.374026:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 00000100:00000001:0.0:1773941581.374029:0:2393:0:(events.c:34:request_out_callback()) Process entered 10000000:00000040:1.0:1773941581.374030:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941581.374035:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000100:00000200:0.0:1773941581.374039:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c1500 x1860112291493248/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941591 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941581.374050:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941581.374054:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c1500 x1860112291493248/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941591 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941581.374066:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941581.374068:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1773941581.374073:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941581.374082:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941581.374085:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1773941581.374088:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941581.374090:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941581.374093:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941581.374095:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1773941581.375332:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941581.375340:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941581.375343:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941581.375346:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941581.375350:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941581.375354:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941581.375358:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941581.375367:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941581.375370:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941581.375372:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324607580 00000400:00000200:0.0:1773941581.375375:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941581.375378:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0x1049 [1] + 0 00000800:00000001:0.0:1773941581.375382:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941581.375387:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941581.375389:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941581.375393:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941581.375396:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941581.375400:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c1500 x1860112291493248/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941591 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941581.375408:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c1500 x1860112291493248/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941591 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941581.375421:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941581.375425:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941581.375427:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941581.375444:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941581.375446:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941581.375450:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941581.375452:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941581.375453:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941581.375457:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941581.375458:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941581.375460:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941581.375465:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c1500 x1860112291493248/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941591 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941581.375472:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c1500 x1860112291493248/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941591 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941581.375477:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941581.375478:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941581.375480:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941581.375482:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941581.375484:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941581.375486:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941581.375487:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941581.375492:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=82 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941581.375495:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941581.375496:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941581.375498:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941581.375500:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941581.375502:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941581.375503:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941581.375512:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c1500 x1860112291493248/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941591 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941581.375517:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941581.375519:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941581.375521:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941581.375522:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941581.375527:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941581.375530:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941581.375531:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941581.375533:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941581.375536:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941581.375537:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941581.375539:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941581.375542:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941581 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941581.375543:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941581.375549:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941581.375550:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941581.375553:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941581.375554:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941581.375555:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941581.375558:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941581.375559:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941581.375562:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941581.375564:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941581.375567:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941581.375569:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941581.375812:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941581.375817:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941581.375824:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c1500 x1860112291493248/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941591 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941581.375826:0:14787:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00080000:3.0:1773941581.375836:0:14787:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:2.0:1773941581.375837:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c1500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291493248:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00100000:3.0:1773941581.375841:0:14787:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:3.0:1773941581.375843:0:14787:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:2.0:1773941581.375844:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 10000000:01000000:3.0:1773941581.375845:0:14787:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000001:2.0:1773941581.375848:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000001:00000001:3.0:1773941581.375850:0:14787:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000040:2.0:1773941581.375860:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c1500 x1860112291493248/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941591 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000001:00000001:3.0:1773941581.375861:0:14787:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941581.375866:0:14787:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941581.375870:0:14787:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:2.0:1773941581.375872:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:3.0:1773941581.375873:0:14787:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941581.375876:0:14787:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941581.375877:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 10000000:00000001:3.0:1773941581.375878:0:14787:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941581.375881:0:14787:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000010:2.0:1773941581.375881:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03523800. 00000100:00080000:3.0:1773941581.375887:0:14787:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 02000000:00000001:2.0:1773941581.375887:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941581.375889:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:3.0:1773941581.375891:0:14787:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000020:00000040:2.0:1773941581.375893:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941581.375894:0:14787:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941581.375897:0:14787:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941581.375899:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00080000:3.0:1773941581.375900:0:14787:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 02000000:00000010:2.0:1773941581.375903:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d1118a400. 00000100:00000001:3.0:1773941581.375904:0:14787:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941581.375906:0:14787:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:3.0:1773941581.375910:0:14787:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 02000000:00000001:2.0:1773941581.375911:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941581.375915:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941581.375917:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c1500. 00000100:00000001:2.0:1773941581.375922:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00080000:3.0:1773941581.375923:0:14787:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:00000001:2.0:1773941581.375925:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:3.0:1773941581.375926:0:14787:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941581.375930:0:14787:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941581.375931:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:02000000:3.0:1773941581.375936:0:14787:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:2.0:1773941581.375940:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941581.375945:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941581.375946:0:14787:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941581.375950:0:14787:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:2.0:1773941581.375951:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:3.0:1773941581.375952:0:14787:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941581.375956:0:14787:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941581.375956:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941581.375992:0:14787:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941581.380008:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d04dd4e00. 00010000:00000010:1.1:1773941581.380016:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d01488800. 00000020:00000001:2.0:1773941581.819187:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941581.819194:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941581.819199:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1773941582.075146:0:461:0:(pinger.c:229:ptlrpc_pinger_process_import()) c1c1ff21-5f68-4bf6-bdc4-258644b24465->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1773941582.075161:0:461:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1773941582.075174:0:461:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06506a00. 00000020:00000040:2.0:1773941582.075183:0:461:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941582.075190:0:461:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941582.075194:0:461:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941582.075202:0:461:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941582.075214:0:461:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff988d0a0db000. 00000100:00000001:2.0:1773941582.075223:0:461:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941582.075237:0:461:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging c1c1ff21-5f68-4bf6-bdc4-258644b24465->MGS req@ffff988d06506a00 x1860112291493376/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941582.075249:0:461:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941582.075270:0:461:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941582.075278:0:461:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6b603d8] to pc [ptlrpcd_01_01+1] req@ffff988d06506a00 x1860112291493376/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941582.075318:0:461:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941582.075323:0:461:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (310) 00000100:00000001:2.0:1773941582.075340:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:2.0:1773941582.075344:0:2403:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00000001:2.0:1773941582.075347:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941582.075355:0:2403:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941582.075358:0:2403:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941582.075361:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941582.075363:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941582.075364:0:2403:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941582.075370:0:2403:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1773941582.075371:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941582.075390:0:2403:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06506a00 x1860112291493376/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941582.075404:0:2403:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941582.075407:0:2403:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941582.075411:0:2403:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941582.075414:0:2403:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941582.075417:0:2403:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941582.075421:0:2403:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2403:1860112291493376:192.168.204.114@tcp:400:kworker.0 00000100:00000001:2.0:1773941582.075425:0:2403:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941582.075430:0:2403:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941582.075433:0:2403:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941582.075436:0:2403:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941582.075442:0:2403:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff988d03356200. 02000000:00000001:2.0:1773941582.075444:0:2403:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941582.075452:0:2403:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941582.075457:0:2403:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000400:00000010:2.0:1773941582.075463:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000200:2.0:1773941582.075471:0:2403:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112291493376, portal 25 00000100:00000001:2.0:1773941582.075473:0:2403:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941582.075475:0:2403:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584960 : -113743512966656 : ffff988d06506a00) 00000100:00000040:2.0:1773941582.075480:0:2403:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06506a00 x1860112291493376/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941598 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941582.075495:0:2403:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941582.075498:0:2403:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:2.0:1773941582.075501:0:2403:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045163b8. 00000100:00000200:2.0:1773941582.075505:0:2403:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1860112291493376, offset 0 00000400:00000200:2.0:1773941582.075511:0:2403:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941582.075520:0:2403:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941582.075528:0:2403:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941582.075533:0:2403:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941582.075537:0:2403:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941582.075542:0:2403:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941582.075544:0:2403:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941582.075548:0:2403:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[132, 131] 00000400:00000200:2.0:1773941582.075552:0:2403:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:2.0:1773941582.075556:0:2403:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [243:243:256:4294967295] 192.168.204.124@tcp LPNI seq info [133:133:8:4294967295] 00000400:00000200:2.0:1773941582.075560:0:2403:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:2.0:1773941582.075570:0:2403:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:2.0:1773941582.075576:0:2403:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:2.0:1773941582.075581:0:2403:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0db900. 00000800:00000200:2.0:1773941582.075588:0:2403:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:2.0:1773941582.075595:0:2403:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:2.0:1773941582.075600:0:2403:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0db900 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1773941582.075616:0:2403:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941582.075620:0:2403:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941582.075623:0:2403:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294973671 00000100:00000001:2.0:1773941582.075632:0:2403:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294972969 00000100:00000001:2.0:1773941582.075661:0:2403:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294973671 00000100:00000001:2.0:1773941582.075669:0:2403:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294972969 00000100:00000001:2.0:1773941582.075674:0:2403:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941582.075679:0:2403:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941582.075710:0:2403:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941582.075716:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941582.075780:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941582.075788:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0db900. 00000400:00000200:0.0:1773941582.075795:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941582.075804:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941582.075810:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045163b8 00000400:00000010:0.0:1773941582.075813:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045163b8. 00000100:00000001:0.0:1773941582.075818:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941582.075825:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06506a00 x1860112291493376/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941598 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941582.075836:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941582.075839:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06506a00 x1860112291493376/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941598 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941582.075845:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941582.075846:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941582.077224:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941582.077252:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941582.077256:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941582.077259:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941582.077267:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941582.077274:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941582.077282:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941582.077288:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941582.077294:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941582.077300:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324607600 00000400:00000200:0.0:1773941582.077306:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941582.077313:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0x1051 [1] + 224 00000800:00000001:0.0:1773941582.077323:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941582.077334:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941582.077338:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941582.077342:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941582.077348:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941582.077355:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06506a00 x1860112291493376/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941598 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941582.077370:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff988d06506a00 x1860112291493376/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941598 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941582.077392:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941582.077400:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941582.077404:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941582.077464:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941582.077469:0:2403:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941582.077475:0:2403:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941582.077477:0:2403:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941582.077481:0:2403:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941582.077487:0:2403:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941582.077489:0:2403:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941582.077493:0:2403:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941582.077501:0:2403:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06506a00 x1860112291493376/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941598 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941582.077513:0:2403:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06506a00 x1860112291493376/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941598 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941582.077521:0:2403:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941582.077523:0:2403:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941582.077526:0:2403:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941582.077528:0:2403:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941582.077531:0:2403:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941582.077533:0:2403:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941582.077535:0:2403:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941582.077542:0:2403:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=83 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941582.077546:0:2403:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941582.077548:0:2403:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941582.077550:0:2403:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941582.077553:0:2403:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:2.0:1773941582.077556:0:2403:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1773941582.077560:0:2403:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1773941582.077565:0:2403:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1773941582.077568:0:2403:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941582.077571:0:2403:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941582.077573:0:2403:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941582.077576:0:2403:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:2.0:1773941582.077590:0:2403:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d06506a00 x1860112291493376/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941598 ref 1 fl Rpc:RNQU/200/0 rc 0/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941582.077598:0:2403:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941582.077600:0:2403:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941582.077602:0:2403:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941582.077605:0:2403:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941582.077608:0:2403:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:2.0:1773941582.077610:0:2403:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941582.077612:0:2403:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941582.077613:0:2403:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941582.077614:0:2403:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941582.077616:0:2403:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941582.077617:0:2403:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941582.077620:0:2403:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1773941582.077621:0:2403:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1773941582.077624:0:2403:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941582.077627:0:2403:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 304 00000400:00000200:2.0:1773941582.077632:0:2403:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941582.077659:0:2403:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:2.0:1773941582.077666:0:2403:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941582.077671:0:2403:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941582.077674:0:2403:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941582.077678:0:2403:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:2.0:1773941582.077683:0:2403:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941582.077686:0:2403:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941582.077690:0:2403:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941582.077694:0:2403:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941582.077698:0:2403:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941582.077702:0:2403:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:2.0:1773941582.077706:0:2403:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941582.077709:0:2403:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941582.077712:0:2403:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941582.077715:0:2403:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:2.0:1773941582.077730:0:2403:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:2.0:1773941582.077735:0:2403:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:2.0:1773941582.077739:0:2403:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941582.077743:0:2403:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941582.077748:0:2403:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941582.077752:0:2403:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941582.077756:0:2403:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941582.077759:0:2403:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941582.077764:0:2403:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06506680. 00000020:00000040:2.0:1773941582.077769:0:2403:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:2.0:1773941582.077774:0:2403:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941582.077776:0:2403:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941582.077779:0:2403:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941582.077780:0:2403:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941582.077783:0:2403:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941582.077789:0:2403:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d035a4000. 00000100:00000001:2.0:1773941582.077793:0:2403:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941582.077800:0:2403:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d06506680 x1860112291493504/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941582.077808:0:2403:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941582.077811:0:2403:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941582.077815:0:2403:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000022f6a110] to pc [ptlrpcd_rcv+-1] req@ffff988d06506680 x1860112291493504/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941582.077840:0:2403:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941582.077843:0:2403:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:2.0:1773941582.077845:0:2403:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:2.0:1773941582.077850:0:2403:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06506a00 x1860112291493376/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941598 ref 1 fl Rpc:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941582.077857:0:2403:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941582.077861:0:2403:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941582.077863:0:2403:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:2.0:1773941582.077875:0:2403:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06506a00 x1860112291493376/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941598 ref 1 fl Interpret:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941582.077878:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941582.077884:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941582.077888:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00100000:2.0:1773941582.077893:0:2403:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2403:1860112291493376:192.168.204.114@tcp:400:kworker.0 00000100:00000040:3.0:1773941582.077902:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06506680 x1860112291493504/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941582.077904:0:2403:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941582.077912:0:2403:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941582.077922:0:2403:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06506a00 x1860112291493376/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941598 ref 1 fl Complete:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941582.077924:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941582.077928:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941582.077936:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941582.077940:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941582.077941:0:2403:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941582.077948:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941582.077949:0:2403:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941582.077953:0:2403:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff988d03356200. 02000000:00000001:2.0:1773941582.077957:0:2403:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941582.077959:0:2403:0:(genops.c:1236:class_import_put()) Process entered 00000100:00100000:3.0:1773941582.078002:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06506680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291493504:192.168.204.114@tcp:250:ptlrpcd_01_00.0 00000020:00000040:2.0:1773941582.078003:0:2403:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941582.078012:0:2403:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941582.078013:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000010:2.0:1773941582.078017:0:2403:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff988d0a0db000. 02000000:00000001:3.0:1773941582.078019:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941582.078024:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941582.078025:0:2403:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941582.078026:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1773941582.078029:0:2403:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:2.0:1773941582.078034:0:2403:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06506a00. 02000000:00000010:3.0:1773941582.078035:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03523000. 00000100:00000001:2.0:1773941582.078038:0:2403:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 02000000:00000001:3.0:1773941582.078041:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941582.078043:0:2403:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941582.078048:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941582.078053:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941582.078054:0:2403:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941582.078059:0:2403:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941582.078063:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:3.0:1773941582.078064:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00000100:00000001:2.0:1773941582.078066:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1773941582.078077:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00000100:00000200:3.0:1773941582.078086:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291493504, portal 25 00000100:00000001:3.0:1773941582.078090:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941582.078093:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584064 : -113743512967552 : ffff988d06506680) 00000100:00000040:3.0:1773941582.078102:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06506680 x1860112291493504/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941592 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941582.078115:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941582.078119:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:3.0:1773941582.078123:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acb28. 00000100:00000200:3.0:1773941582.078128:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291493504, offset 0 00000400:00000200:3.0:1773941582.078136:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941582.078143:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941582.078150:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941582.078156:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941582.078160:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941582.078164:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941582.078168:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941582.078172:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[132, 133] 00000400:00000200:3.0:1773941582.078178:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941582.078183:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [244:244:256:4294967295] 192.168.204.114@tcp LPNI seq info [134:134:8:4294967295] 00000400:00000200:3.0:1773941582.078190:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941582.078203:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:3.0:1773941582.078224:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:3.0:1773941582.078230:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74500. 00000800:00000200:3.0:1773941582.078237:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:3.0:1773941582.078246:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:3.0:1773941582.078251:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74500 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:3.0:1773941582.078276:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941582.078283:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:3.0:1773941582.078286:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294973674 00000100:00000001:3.0:1773941582.078292:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294973671 00000100:00000001:3.0:1773941582.078297:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294973674 00000100:00000001:3.0:1773941582.078302:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294973671 00000100:00000001:3.0:1773941582.078306:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941582.078311:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941582.078315:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941582.078319:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941582.078327:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941582.078330:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1773941582.078334:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941582.078337:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941582.078339:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941582.078342:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941582.078394:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941582.078400:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74500. 00000400:00000200:0.0:1773941582.078405:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941582.078411:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941582.078416:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acb28 00000400:00000010:0.0:1773941582.078418:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acb28. 00000100:00000001:0.0:1773941582.078422:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941582.078427:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06506680 x1860112291493504/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941592 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941582.078437:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941582.078448:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06506680 x1860112291493504/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941592 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941582.078456:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941582.078458:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941582.080286:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941582.080299:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941582.080302:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941582.080304:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941582.080310:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941582.080316:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941582.080322:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941582.080326:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941582.080330:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941582.080334:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324607680 00000400:00000200:0.0:1773941582.080338:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941582.080343:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0x1059 [1] + 0 00000800:00000001:0.0:1773941582.080349:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941582.080357:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941582.080360:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941582.080365:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941582.080370:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941582.080376:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06506680 x1860112291493504/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941592 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941582.080388:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d06506680 x1860112291493504/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941592 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941582.080414:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941582.080423:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941582.080429:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941582.080450:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941582.080453:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941582.080458:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941582.080460:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941582.080462:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941582.080467:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941582.080469:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941582.080472:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941582.080477:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06506680 x1860112291493504/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941592 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941582.080488:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06506680 x1860112291493504/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941592 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941582.080507:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941582.080510:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941582.080512:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941582.080514:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941582.080515:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941582.080518:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941582.080520:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941582.080524:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=83 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941582.080527:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941582.080529:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941582.080531:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941582.080535:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941582.080537:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941582.080540:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941582.080544:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06506680 x1860112291493504/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941592 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941582.080550:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941582.080552:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941582.080554:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941582.080557:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941582.080562:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941582.080564:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941582.080566:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941582.080568:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941582.080570:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941582.080572:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941582.080574:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941582.080578:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941582 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941582.080581:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941582.080587:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941582.080589:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941582.080591:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941582.080593:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941582.080599:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941582.080603:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941582.080605:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941582.080608:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941582.080611:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941582.080615:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941582.080617:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941582.080983:0:14788:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941582.080990:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1773941582.080994:0:14788:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00000001:2.0:1773941582.080998:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941582.081000:0:14788:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000040:2.0:1773941582.081011:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06506680 x1860112291493504/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941592 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941582.081025:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06506680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291493504:192.168.204.114@tcp:250:ptlrpcd_01_00.0 00000100:00000001:2.0:1773941582.081036:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941582.081039:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1773941582.081048:0:14788:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941582.081052:0:14788:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000040:2.0:1773941582.081055:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06506680 x1860112291493504/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941592 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ptlrpcd_01_00.0' uid:0 gid:0 projid:4294967295 00000001:00000001:3.0:1773941582.081057:0:14788:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941582.081063:0:14788:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941582.081069:0:14788:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941582.081071:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:3.0:1773941582.081073:0:14788:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:2.0:1773941582.081077:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:3.0:1773941582.081078:0:14788:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000010:2.0:1773941582.081084:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03523000. 00000001:00000001:3.0:1773941582.081085:0:14788:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941582.081092:0:14788:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941582.081094:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1773941582.081095:0:14788:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000001:2.0:1773941582.081101:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000100:00080000:3.0:1773941582.081104:0:14788:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000020:00000040:2.0:1773941582.081105:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941582.081114:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941582.081115:0:14788:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 02000000:00000010:2.0:1773941582.081118:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d035a4000. 00000100:00000001:3.0:1773941582.081120:0:14788:0:(recover.c:147:ptlrpc_resend()) Process entered 02000000:00000001:2.0:1773941582.081124:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1773941582.081125:0:14788:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941582.081127:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:3.0:1773941582.081131:0:14788:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000010:2.0:1773941582.081132:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06506680. 00000100:00000001:3.0:1773941582.081140:0:14788:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:2.0:1773941582.081141:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 10000000:01000000:3.0:1773941582.081143:0:14788:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000100:00000001:2.0:1773941582.081146:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000040:3.0:1773941582.081148:0:14788:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00000001:2.0:1773941582.081152:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941582.081158:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941582.081160:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941582.081162:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941582.081164:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1773941582.081168:0:14788:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:3.0:1773941582.081172:0:14788:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941582.081174:0:14788:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:3.0:1773941582.081179:0:14788:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:3.0:1773941582.081185:0:14788:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941582.081187:0:14788:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941582.081198:0:14788:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941582.081200:0:14788:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941582.081202:0:14788:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941582.451140:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941582.451151:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941582.451157:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941582.451160:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941582.451163:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941582.451164:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941582.451168:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941582.451170:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941582.451174:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:1.0:1773941582.451177:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941582.451180:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941582.451198:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c0700. 00000020:00000040:1.0:1773941582.451207:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941582.451215:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941582.451221:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941582.451230:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941582.451244:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d181fd600. 00000100:00000001:1.0:1773941582.451256:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941582.451265:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941582.451269:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941582.451282:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d04dd4e00. 00010000:00000001:1.0:1773941582.451294:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941582.451337:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d01488800. 00000020:00000001:1.0:1773941582.451360:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941582.451364:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d01488800 with handle 0x8e86767622e2a55 to hash 00000020:00000001:1.0:1773941582.451366:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941582.451368:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941582.451371:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941582.451377:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941582.451380:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941582.451386:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a55 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941582.451397:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a55 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941582.451413:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a55 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941582.451419:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941582.451421:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941582.451430:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941582.451432:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330528610752 : -113743180940864 : ffff988d1a1ab9c0) 00000100:00000001:1.0:1773941582.451436:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941582.451438:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00000100:00000001:1.0:1773941582.451445:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941582.451451:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941582.451453:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941582.451456:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941582.451486:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c0700 x1860112291493632/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941582.451496:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941582.451499:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941582.451503:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941582.451507:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941582.451515:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941582.451520:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291493632:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941582.451525:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941582.451533:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941582.451536:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941582.451539:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941582.451544:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d11189400. 02000000:00000001:1.0:1773941582.451547:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941582.451557:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941582.451562:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c5d8. 00000400:00000010:1.0:1773941582.451571:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000100:00000200:1.0:1773941582.451581:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291493632, portal 25 00000100:00000001:1.0:1773941582.451584:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941582.451585:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00000100:00000040:1.0:1773941582.451591:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c0700 x1860112291493632/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941598 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941582.451599:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941582.451603:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941582.451605:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de3b8. 00000100:00000200:1.0:1773941582.451608:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291493632, offset 0 00000400:00000200:1.0:1773941582.451616:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941582.451628:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941582.451673:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941582.451697:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941582.451702:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941582.451707:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941582.451710:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941582.451713:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[134, 133] 00000400:00000200:1.0:1773941582.451717:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:1.0:1773941582.451722:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [245:245:256:4294967295] 192.168.204.124@tcp LPNI seq info [135:135:8:4294967295] 00000400:00000200:1.0:1773941582.451727:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941582.451739:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:1.0:1773941582.451752:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:1.0:1773941582.451758:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfd900. 00000800:00000200:1.0:1773941582.451767:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:1.0:1773941582.451777:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:1.0:1773941582.451782:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfd900 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941582.451805:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941582.451810:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941582.451813:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294974047 00000100:00000001:1.0:1773941582.451824:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294973674 00000100:00000001:1.0:1773941582.451831:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294974047 00000100:00000001:1.0:1773941582.451836:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294973674 00000100:00000001:1.0:1773941582.451840:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941582.451843:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941582.451846:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941582.451849:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941582.451852:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 11 seconds 00000100:00000001:1.0:1773941582.451856:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941582.451860:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941582.452053:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941582.452063:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfd900. 00000400:00000200:0.0:1773941582.452071:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941582.452082:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941582.452109:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de3b8 00000400:00000010:0.0:1773941582.452112:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de3b8. 00000100:00000001:0.0:1773941582.452118:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941582.452128:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c0700 x1860112291493632/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941598 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941582.452147:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941582.452154:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d203c0700 x1860112291493632/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941598 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941582.452167:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941582.452170:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941582.453815:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941582.453832:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941582.453835:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941582.453838:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941582.453846:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941582.453854:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941582.453867:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941582.453873:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941582.453881:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941582.453887:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324607700 00000400:00000200:0.0:1773941582.453895:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941582.453905:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0x1061 [1] + 224 00000800:00000001:0.0:1773941582.453919:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941582.453932:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941582.453936:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941582.453944:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941582.453952:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941582.453987:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c0700 x1860112291493632/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941598 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941582.454000:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d203c0700 x1860112291493632/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941598 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941582.454019:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941582.454038:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941582.454043:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941582.454105:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941582.454116:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941582.454118:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941582.454122:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:1.0:1773941582.454128:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:1.0:1773941582.454131:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:1.0:1773941582.454134:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941582.454142:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c0700 x1860112291493632/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941598 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941582.454156:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c0700 x1860112291493632/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941598 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941582.454164:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941582.454166:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941582.454196:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941582.454199:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941582.454203:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941582.454206:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941582.454210:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941582.454219:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=83 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941582.454224:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=0 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941582.454227:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941582.454232:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:1.0:1773941582.454236:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941582.454240:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941582.454243:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941582.454248:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941582.454253:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941582.454258:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941582.454260:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941582.454263:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941582.454265:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941582.454270:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d203c0700 x1860112291493632/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941598 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941582.454280:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941582.454281:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941582.454284:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941582.454288:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941582.454291:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941582.454293:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941582.454295:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941582.454297:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941582.454298:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941582.454300:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941582.454301:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941582.454304:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941582.454306:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941582.454310:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941582.454318:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 305 00000400:00000200:1.0:1773941582.454325:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941582.454331:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941582.454336:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941582.454340:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941582.454344:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941582.454346:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941582.454351:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941582.454353:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941582.454356:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941582.454359:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941582.454362:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941582.454365:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941582.454368:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941582.454370:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941582.454372:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941582.454374:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941582.454378:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941582.454382:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941582.454385:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941582.454388:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941582.454392:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941582.454395:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941582.454398:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941582.454400:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941582.454406:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c0a80. 00000020:00000040:1.0:1773941582.454410:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941582.454415:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941582.454417:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941582.454420:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941582.454423:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941582.454428:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941582.454461:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d11188400. 00000100:00000001:1.0:1773941582.454469:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941582.454478:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c0a80 x1860112291493760/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941582.454489:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941582.454495:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941582.454500:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000431867b8] to pc [ptlrpcd_rcv+-1] req@ffff988d203c0a80 x1860112291493760/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941582.454538:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941582.454543:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941582.454547:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941582.454552:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941582.454558:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941582.454561:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941582.454566:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941582.454572:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941582.454577:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941582.454590:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c0700 x1860112291493632/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941598 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941582.454613:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941582.454625:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941582.454627:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941582.454651:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941582.454654:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941582.454663:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000040:2.0:1773941582.454664:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c0a80 x1860112291493760/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941582.454670:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941582.454686:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941582.454691:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941582.454693:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c0700 x1860112291493632/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941598 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1773941582.454696:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941582.454701:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941582.454708:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941582.454719:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291493760:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941582.454728:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941582.454735:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:1.0:1773941582.454735:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291493632:192.168.204.114@tcp:101:ll_cfg_requeue.0 02000000:00000001:2.0:1773941582.454741:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941582.454747:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1773941582.454749:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941582.454754:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941582.454757:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941582.454763:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000010:2.0:1773941582.454767:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a4000. 02000000:00000001:2.0:1773941582.454772:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941582.454782:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000040:1.0:1773941582.454789:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c0700 x1860112291493632/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941598 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1773941582.454792:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000400:00000010:2.0:1773941582.454805:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000001:1.0:1773941582.454811:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1773941582.454817:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291493760, portal 25 00000100:00000010:1.0:1773941582.454820:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1a1ab9c0. 00000100:00000001:2.0:1773941582.454825:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941582.454830:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:2.0:1773941582.454832:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631457408 : -113743078094208 : ffff988d203c0a80) 00000100:00000001:1.0:1773941582.454837:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:1.0:1773941582.454846:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000040:2.0:1773941582.454848:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c0a80 x1860112291493760/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941592 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941582.454855:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941582.454859:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941582.454865:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488800 refcount=4 00000100:00000001:2.0:1773941582.454871:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:1.0:1773941582.454872:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00000100:00000040:2.0:1773941582.454880:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00000001:1.0:1773941582.454881:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00000400:00000010:2.0:1773941582.454886:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516f68. 00010000:00010000:1.0:1773941582.454891:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a55 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:2.0:1773941582.454893:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291493760, offset 0 00000400:00000200:2.0:1773941582.454910:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00000001:1.0:1773941582.454914:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000200:2.0:1773941582.454921:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:1.0:1773941582.454922:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a55 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941582.454933:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:1.0:1773941582.454935:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941582.454943:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a55 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941582.454944:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941582.454954:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941582.454996:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941582.455018:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941582.455023:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[134, 135] 00010000:00000001:1.0:1773941582.455026:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:2.0:1773941582.455032:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00010000:00000001:1.0:1773941582.455033:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941582.455040:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a55 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941582.455041:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [246:246:256:4294967295] 192.168.204.114@tcp LPNI seq info [136:136:8:4294967295] 00000400:00000200:2.0:1773941582.455065:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941582.455070:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941582.455077:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941582.455087:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:2.0:1773941582.455088:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00010000:00010000:1.0:1773941582.455114:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a55 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941582.455119:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:2.0:1773941582.455126:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0db000. 00000800:00000200:2.0:1773941582.455140:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00010000:00010000:1.0:1773941582.455141:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d01488800 already unused, calling callback (ffffffffc12b4c20) 10000000:00000001:1.0:1773941582.455148:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941582.455153:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a55 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941582.455154:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:2.0:1773941582.455164:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0db000 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941582.455169:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941582.455171:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941582.455173:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941582.455178:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488800 refcount=5 00000020:00000001:1.0:1773941582.455201:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941582.455203:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941582.455207:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941582.455211:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a55 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941582.455214:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941582.455227:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:2.0:1773941582.455228:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 10000000:00010000:1.0:1773941582.455236:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a55 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941582.455237:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294974051 10000000:01000000:1.0:1773941582.455254:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000100:00000001:2.0:1773941582.455256:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294974047 10000000:00000040:1.0:1773941582.455258:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941582.455265:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941582.455269:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294974051 00010000:00010000:1.0:1773941582.455271:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941582.455278:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000100:00000001:2.0:1773941582.455281:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294974047 00010000:00000001:1.0:1773941582.455284:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000100:00000001:2.0:1773941582.455288:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941582.455290:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000100:00000001:2.0:1773941582.455298:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941582.455299:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:2.0:1773941582.455309:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941582.455310:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941582.455322:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d01488800 with handle 0x8e86767622e2a55 from hash 00000100:00000001:2.0:1773941582.455323:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:1.0:1773941582.455326:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941582.455333:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941582.455334:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941582.455340:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941582.455349:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00010000:00000001:1.0:1773941582.455350:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941582.455355:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000100:00000001:2.0:1773941582.455360:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:1.0:1773941582.455361:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000100:00000001:2.0:1773941582.455368:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00010000:00000001:1.0:1773941582.455370:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000200:0.0:1773941582.455375:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00000001:1.0:1773941582.455377:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:2.0:1773941582.455378:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941582.455383:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1773941582.455387:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0db000. 00000100:00000001:2.0:1773941582.455391:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941582.455391:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941582.455398:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00010000:1.0:1773941582.455405:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a55 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:0.0:1773941582.455418:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00010000:00000001:1.0:1773941582.455422:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941582.455427:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941582.455429:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000400:00000200:0.0:1773941582.455431:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516f68 00010000:00000001:1.0:1773941582.455433:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941582.455440:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000010:0.0:1773941582.455441:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516f68. 00010000:00000001:1.0:1773941582.455442:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941582.455447:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941582.455455:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a55 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941582.455455:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941582.455471:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c0a80 x1860112291493760/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941592 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000040:1.0:1773941582.455474:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d04dd4e00 count: 0 00010000:00000001:1.0:1773941582.455493:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:0.0:1773941582.455496:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00010000:00000001:1.0:1773941582.455498:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941582.455506:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941582.455513:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c0a80 x1860112291493760/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941592 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941582.455518:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c0700 x1860112291493632/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941598 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941582.455583:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1773941582.455586:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941582.455588:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1773941582.455595:0:2393:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000010:1.0:1773941582.455597:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d11189400. 02000000:00000001:1.0:1773941582.455604:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941582.455607:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941582.455609:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941582.455613:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941582.455616:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d181fd600. 02000000:00000001:1.0:1773941582.455620:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941582.455622:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941582.455651:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c0700. 00000100:00000001:1.0:1773941582.455656:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941582.455658:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941582.455661:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941582.455664:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941582.455668:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941582.455672:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d04d67c80. 00000020:00000010:1.0:1773941582.455677:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d181fd600. 00000020:00000010:1.0:1773941582.455682:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d181fc400. 00000040:00000010:1.0:1773941582.455687:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d181fd000. 00000020:00000010:1.0:1773941582.455693:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941582.455698:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d181fce00. 80000000:00000010:1.0:1773941582.455703:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d054f37e0. 00040000:00000010:1.0:1773941582.455710:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d181fc800. 20000000:00000010:1.0:1773941582.455718:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d181fd800. 10000000:00000040:1.0:1773941582.455726:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:1.0:1773941582.455729:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941582.455736:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d181fc400. 00000040:00000010:1.0:1773941582.455741:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d181fd000. 00000020:00000010:1.0:1773941582.455746:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941582.455752:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d181fce00. 80000000:00000010:1.0:1773941582.455756:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d054f37e0. 00040000:00000010:1.0:1773941582.455762:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d181fc800. 20000000:00000010:1.0:1773941582.455767:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d181fd800. 00000020:00000010:1.0:1773941582.455771:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d181fd600. 10000000:00000010:1.0:1773941582.455776:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d04d67c80. 10000000:01000000:1.0:1773941582.455780:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941582.455786:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941582.455790:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941582.455794:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941582.455797:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941582.455800:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941582.455804:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941582.457559:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941582.457594:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941582.457598:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941582.457605:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941582.457612:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941582.457621:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941582.457658:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941582.457665:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941582.457672:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941582.457678:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324607780 00000400:00000200:0.0:1773941582.457694:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941582.457701:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0x1071 [1] + 0 00000800:00000001:0.0:1773941582.457710:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941582.457720:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941582.457724:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941582.457730:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941582.457736:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941582.457762:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c0a80 x1860112291493760/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941592 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941582.457781:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c0a80 x1860112291493760/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941592 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941582.457809:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941582.457818:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941582.457824:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941582.457863:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941582.457868:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941582.457874:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941582.457875:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941582.457878:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941582.457884:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941582.457886:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941582.457889:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941582.457897:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c0a80 x1860112291493760/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941592 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941582.457921:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c0a80 x1860112291493760/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941592 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941582.457929:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941582.457931:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941582.457933:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941582.457935:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941582.457937:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941582.457939:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941582.457951:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941582.457956:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=83 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941582.457992:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=0 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941582.457996:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941582.457998:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941582.458003:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941582.458005:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941582.458008:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941582.458015:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c0a80 x1860112291493760/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941592 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941582.458024:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941582.458027:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941582.458030:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941582.458033:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941582.458039:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941582.458041:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941582.458043:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941582.458045:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941582.458048:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941582.458050:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941582.458053:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:2.0:1773941582.458058:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941582 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941582.458061:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941582.458068:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:2.0:1773941582.458070:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941582.458072:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941582.458075:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941582.458077:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941582.458081:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:2.0:1773941582.458084:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941582.458087:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941582.458090:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941582.458094:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:2.0:1773941582.458104:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941582.458462:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941582.458468:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941582.458478:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c0a80 x1860112291493760/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941592 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941582.458491:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291493760:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941582.458499:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941582.458502:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941582.458507:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c0a80 x1860112291493760/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941592 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941582.458516:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941582.458518:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1773941582.458520:0:14789:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 02000000:00000010:3.0:1773941582.458522:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a4000. 02000000:00000001:3.0:1773941582.458530:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941582.458532:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941582.458534:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941582.458538:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00080000:2.0:1773941582.458541:0:14789:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 02000000:00000010:3.0:1773941582.458542:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d11188400. 02000000:00000001:3.0:1773941582.458550:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00100000:2.0:1773941582.458554:0:14789:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000001:3.0:1773941582.458555:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941582.458560:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c0a80. 00000100:00000001:2.0:1773941582.458563:0:14789:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:3.0:1773941582.458566:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 10000000:01000000:2.0:1773941582.458570:0:14789:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000001:3.0:1773941582.458571:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941582.458578:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:2.0:1773941582.458582:0:14789:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:3.0:1773941582.458585:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941582.458590:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:2.0:1773941582.458590:0:14789:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:3.0:1773941582.458599:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941582.458601:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:2.0:1773941582.458602:0:14789:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:2.0:1773941582.458610:0:14789:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941582.458612:0:14789:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941582.458617:0:14789:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941582.458620:0:14789:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941582.458624:0:14789:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941582.458654:0:14789:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941582.458667:0:14789:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941582.458671:0:14789:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:2.0:1773941582.458674:0:14789:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1773941582.458678:0:14789:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941582.458682:0:14789:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941582.458685:0:14789:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:2.0:1773941582.458688:0:14789:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:2.0:1773941582.458713:0:14789:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:2.0:1773941582.458716:0:14789:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941582.458720:0:14789:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:2.0:1773941582.458735:0:14789:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:2.0:1773941582.458741:0:14789:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941582.458746:0:14789:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941582.458749:0:14789:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941582.458754:0:14789:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941582.458757:0:14789:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941582.462023:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d04dd4e00. 00010000:00000010:1.1:1773941582.462034:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d01488800. 00000020:00000001:2.0:1773941582.843243:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941582.843250:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941582.843256:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941583.739070:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294974051, at 4294974080 00000100:00000040:0.0:1773941583.739071:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294974051, at 4294974080 00000100:00000010:0.0:1773941583.739088:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3840. 00000100:00000010:1.0:1773941583.739095:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3720. 10000000:00000001:1.0:1773941583.867114:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941583.867121:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941583.867125:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941583.867128:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941583.867130:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941583.867131:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941583.867133:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941583.867136:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941583.867139:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:1.0:1773941583.867143:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941583.867144:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941583.867155:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c0700. 00000020:00000040:1.0:1773941583.867159:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941583.867164:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000100:00000001:1.0:1773941583.867164:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941583.867168:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941583.867174:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941583.867180:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d181fd800. 00000100:00000001:1.0:1773941583.867186:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941583.867190:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00000020:00000001:2.0:1773941583.867191:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941583.867194:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00010000:2.0:1773941583.867199:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000010:1.0:1773941583.867204:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d04dd4e00. 00010000:00000001:1.0:1773941583.867210:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941583.867213:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d01488800. 00000020:00000001:1.0:1773941583.867217:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941583.867219:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d01488800 with handle 0x8e86767622e2a5c to hash 00000020:00000001:1.0:1773941583.867221:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941583.867222:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941583.867224:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941583.867228:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941583.867231:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941583.867233:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a5c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941583.867241:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a5c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941583.867264:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a5c lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941583.867268:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941583.867270:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941583.867275:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941583.867277:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330528610752 : -113743180940864 : ffff988d1a1ab9c0) 00000100:00000001:1.0:1773941583.867279:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941583.867280:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00000100:00000001:1.0:1773941583.867284:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941583.867287:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941583.867289:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941583.867291:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941583.867299:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c0700 x1860112291493888/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941583.867306:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941583.867308:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941583.867310:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941583.867313:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941583.867314:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941583.867317:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291493888:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941583.867320:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941583.867325:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941583.867326:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941583.867328:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941583.867332:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d11189400. 02000000:00000001:1.0:1773941583.867333:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941583.867340:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941583.867343:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c5d8. 00000400:00000010:1.0:1773941583.867348:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000100:00000200:1.0:1773941583.867354:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291493888, portal 25 00000100:00000001:1.0:1773941583.867355:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941583.867359:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00000100:00000040:1.0:1773941583.867363:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c0700 x1860112291493888/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941599 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941583.867369:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941583.867371:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941583.867373:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de110. 00000100:00000200:1.0:1773941583.867375:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291493888, offset 0 00000400:00000200:1.0:1773941583.867379:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941583.867387:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941583.867393:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941583.867397:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941583.867400:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941583.867403:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941583.867405:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941583.867407:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[136, 135] 00000400:00000200:1.0:1773941583.867410:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:1.0:1773941583.867413:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [247:247:256:4294967295] 192.168.204.124@tcp LPNI seq info [137:137:8:4294967295] 00000400:00000200:1.0:1773941583.867417:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941583.867425:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:1.0:1773941583.867430:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:1.0:1773941583.867433:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfd200. 00000800:00000200:1.0:1773941583.867439:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:1.0:1773941583.867446:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:1.0:1773941583.867449:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfd200 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941583.867462:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941583.867465:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941583.867468:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294975463 00000100:00000010:1.0:1773941583.867471:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3720. 00000100:00000001:1.0:1773941583.867476:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1773941583.867481:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294975463 00000100:00000010:1.0:1773941583.867487:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f37e0. 00000100:00000001:1.0:1773941583.867489:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1773941583.867491:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941583.867493:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941583.867495:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941583.867497:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941583.867499:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 11 seconds 00000100:00000001:1.0:1773941583.867501:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941583.867504:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941583.867626:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941583.867632:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfd200. 00000400:00000200:0.0:1773941583.867637:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941583.867643:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941583.867649:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de110 00000400:00000010:0.0:1773941583.867652:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de110. 00000100:00000001:0.0:1773941583.867657:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941583.867663:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c0700 x1860112291493888/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941599 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941583.867674:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941583.867678:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d203c0700 x1860112291493888/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941599 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941583.867686:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941583.867689:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941583.868686:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941583.868700:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941583.868702:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941583.868704:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941583.868708:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941583.868712:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941583.868717:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941583.868720:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941583.868723:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941583.868732:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324607800 00000400:00000200:0.0:1773941583.868735:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941583.868739:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0x1079 [1] + 224 00000800:00000001:0.0:1773941583.868746:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941583.868753:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941583.868754:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941583.868757:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941583.868760:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941583.868764:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c0700 x1860112291493888/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941599 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941583.868774:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d203c0700 x1860112291493888/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941599 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941583.868791:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941583.868797:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941583.868800:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941583.868843:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941583.868850:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941583.868852:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941583.868854:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:1.0:1773941583.868858:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:1.0:1773941583.868861:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:1.0:1773941583.868864:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941583.868870:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c0700 x1860112291493888/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941599 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941583.868880:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c0700 x1860112291493888/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941599 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941583.868887:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941583.868889:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941583.868892:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941583.868894:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941583.868897:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941583.868899:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941583.868902:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941583.868908:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=84 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941583.868911:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941583.868914:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941583.868918:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:1.0:1773941583.868921:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941583.868924:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941583.868927:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941583.868938:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941583.868943:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941583.868948:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941583.868950:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941583.868952:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941583.868955:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941583.868979:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d203c0700 x1860112291493888/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941599 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941583.868985:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941583.868986:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941583.868988:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941583.868990:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941583.868992:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941583.868994:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941583.868995:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941583.868996:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941583.868996:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941583.868997:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941583.868998:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941583.869000:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941583.869001:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941583.869004:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941583.869006:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 305 00000400:00000200:1.0:1773941583.869010:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941583.869013:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941583.869016:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941583.869018:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941583.869020:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941583.869022:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941583.869024:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941583.869026:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941583.869028:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941583.869036:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941583.869038:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941583.869039:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941583.869042:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941583.869043:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941583.869045:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941583.869046:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941583.869048:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941583.869050:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941583.869053:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941583.869055:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941583.869059:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941583.869061:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941583.869064:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941583.869066:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941583.869070:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c3100. 00000020:00000040:1.0:1773941583.869073:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941583.869076:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941583.869078:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941583.869081:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941583.869083:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941583.869085:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941583.869090:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d1118b400. 00000100:00000001:1.0:1773941583.869093:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941583.869099:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c3100 x1860112291494016/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941583.869107:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941583.869109:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941583.869113:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007ece65f9] to pc [ptlrpcd_rcv+-1] req@ffff988d203c3100 x1860112291494016/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941583.869129:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941583.869131:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941583.869137:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941583.869139:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941583.869142:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941583.869143:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941583.869144:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941583.869147:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941583.869150:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941583.869155:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c0700 x1860112291493888/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941599 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941583.869162:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941583.869162:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941583.869167:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941583.869168:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941583.869171:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941583.869172:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941583.869174:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941583.869182:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c0700 x1860112291493888/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941599 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941583.869188:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c3100 x1860112291494016/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941583.869220:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291493888:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941583.869225:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941583.869226:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941583.869227:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941583.869229:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941583.869231:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c0700 x1860112291493888/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941599 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941583.869291:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941583.869297:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941583.869299:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941583.869302:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941583.869303:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:1.0:1773941583.869305:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941583.869312:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941583.869313:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941583.869315:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488800 refcount=4 00000100:00000001:3.0:1773941583.869316:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941583.869318:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 02000000:00000001:3.0:1773941583.869324:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00010000:00000001:1.0:1773941583.869326:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 02000000:00000001:3.0:1773941583.869331:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00010000:00010000:1.0:1773941583.869332:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a5c lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941583.869344:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941583.869345:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00010000:00010000:1.0:1773941583.869357:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a5c lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1773941583.869358:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291494016:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941583.869380:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941583.869390:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941583.869392:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941583.869394:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:1.0:1773941583.869398:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 02000000:00000010:3.0:1773941583.869401:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03522800. 00010000:00010000:1.0:1773941583.869402:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a5c lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941583.869411:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941583.869422:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:3.0:1773941583.869428:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941583.869429:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000010:3.0:1773941583.869434:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00010000:00010000:1.0:1773941583.869436:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a5c lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:3.0:1773941583.869441:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00010000:00000001:1.0:1773941583.869449:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941583.869451:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1773941583.869454:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291494016, portal 25 00010000:00000001:1.0:1773941583.869454:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941583.869460:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a5c lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941583.869462:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941583.869466:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631467264 : -113743078084352 : ffff988d203c3100) 00010000:00010000:1.0:1773941583.869480:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d01488800 already unused, calling callback (ffffffffc12b4c20) 00000100:00000040:3.0:1773941583.869482:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c3100 x1860112291494016/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941593 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000001:1.0:1773941583.869487:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941583.869492:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a5c lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941583.869502:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941583.869512:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00000001:1.0:1773941583.869513:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000400:00000010:3.0:1773941583.869516:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac2a8. 00010000:00000001:1.0:1773941583.869517:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941583.869527:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000200:3.0:1773941583.869528:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291494016, offset 0 00000020:00000040:1.0:1773941583.869532:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488800 refcount=5 00000400:00000200:3.0:1773941583.869538:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000020:00000001:1.0:1773941583.869546:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00000400:00000200:3.0:1773941583.869552:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941583.869558:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00000400:00000200:3.0:1773941583.869562:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:1.0:1773941583.869569:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:3.0:1773941583.869570:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941583.869576:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00010000:1.0:1773941583.869578:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a5c lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941583.869582:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941583.869586:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 10000000:00000001:1.0:1773941583.869597:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:3.0:1773941583.869598:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[136, 137] 10000000:00010000:1.0:1773941583.869618:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a5c lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941583.869620:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941583.869632:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [248:248:256:4294967295] 192.168.204.114@tcp LPNI seq info [138:138:8:4294967295] 00000400:00000200:3.0:1773941583.869654:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 10000000:01000000:1.0:1773941583.869655:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941583.869661:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941583.869664:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941583.869670:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000400:00000200:3.0:1773941583.869672:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00010000:00000001:1.0:1773941583.869675:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941583.869679:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000800:00000200:3.0:1773941583.869684:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00010000:00000001:1.0:1773941583.869686:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941583.869691:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000800:00000010:3.0:1773941583.869692:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74f00. 00010000:00000001:1.0:1773941583.869694:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941583.869701:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d01488800 with handle 0x8e86767622e2a5c from hash 00010000:00000001:1.0:1773941583.869704:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000800:00000200:3.0:1773941583.869708:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00010000:00000001:1.0:1773941583.869710:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941583.869717:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941583.869720:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941583.869721:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941583.869723:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000800:00000200:3.0:1773941583.869731:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00010000:00000001:1.0:1773941583.869732:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941583.869737:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000200:3.0:1773941583.869739:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74f00 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941583.869739:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941583.869751:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941583.869756:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a5c lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941583.869761:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941583.869762:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941583.869770:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941583.869771:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:3.0:1773941583.869777:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294975465 00010000:00000001:1.0:1773941583.869777:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941583.869781:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941583.869784:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941583.869787:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941583.869789:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941583.869791:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294975463 00010000:00010000:1.0:1773941583.869792:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a5c lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941583.869801:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294975465 00010000:00000040:1.0:1773941583.869810:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d04dd4e00 count: 0 00000100:00000001:3.0:1773941583.869812:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294975463 00000100:00000001:3.0:1773941583.869822:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941583.869826:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941583.869831:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941583.869832:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:3.0:1773941583.869834:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:1.0:1773941583.869835:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000800:00000200:0.0:1773941583.869840:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:1.0:1773941583.869852:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000800:00000010:0.0:1773941583.869852:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74f00. 00000100:00000001:3.0:1773941583.869857:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:0.0:1773941583.869859:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1773941583.869860:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000040:1.0:1773941583.869861:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c0700 x1860112291493888/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941599 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941583.869871:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941583.869878:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000200:0.0:1773941583.869880:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:1.0:1773941583.869881:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1773941583.869887:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941583.869888:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000400:00000200:0.0:1773941583.869890:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac2a8 00000100:00000001:3.0:1773941583.869897:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941583.869898:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d11189400. 00000400:00000010:0.0:1773941583.869898:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac2a8. 02000000:00000001:1.0:1773941583.869909:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1773941583.869910:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:1.0:1773941583.869911:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941583.869914:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941583.869917:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000200:0.0:1773941583.869917:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c3100 x1860112291494016/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941593 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000010:1.0:1773941583.869921:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d181fd800. 02000000:00000001:1.0:1773941583.869926:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941583.869928:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941583.869931:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c0700. 00000100:00000001:0.0:1773941583.869931:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1773941583.869935:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000040:0.0:1773941583.869937:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c3100 x1860112291494016/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941593 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941583.869938:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941583.869949:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941583.869954:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000100:00000001:0.0:1773941583.869984:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941583.869986:0:2393:0:(events.c:68:request_out_callback()) Process leaving 10000000:00000001:1.0:1773941583.869994:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941583.869998:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d04d672c0. 00000020:00000010:1.0:1773941583.870003:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d181fd800. 00000020:00000010:1.0:1773941583.870007:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d181fc800. 00000040:00000010:1.0:1773941583.870010:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d181fce00. 00000020:00000010:1.0:1773941583.870014:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941583.870023:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d181fd000. 80000000:00000010:1.0:1773941583.870027:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d054f39c0. 00040000:00000010:1.0:1773941583.870031:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d181fc400. 20000000:00000010:1.0:1773941583.870035:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d181fc200. 10000000:00000040:1.0:1773941583.870041:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:1.0:1773941583.870043:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941583.870047:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d181fc800. 00000040:00000010:1.0:1773941583.870050:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d181fce00. 00000020:00000010:1.0:1773941583.870052:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941583.870055:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d181fd000. 80000000:00000010:1.0:1773941583.870057:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d054f39c0. 00040000:00000010:1.0:1773941583.870060:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d181fc400. 20000000:00000010:1.0:1773941583.870062:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d181fc200. 00000020:00000010:1.0:1773941583.870064:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d181fd800. 10000000:00000010:1.0:1773941583.870066:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d04d672c0. 10000000:01000000:1.0:1773941583.870068:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941583.870070:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941583.870072:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941583.870073:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941583.870074:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941583.870075:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941583.870076:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941583.871085:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941583.871093:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941583.871095:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941583.871103:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941583.871107:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941583.871111:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941583.871115:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941583.871118:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941583.871121:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941583.871126:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324607880 00000400:00000200:0.0:1773941583.871129:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941583.871133:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0x1089 [1] + 0 00000800:00000001:0.0:1773941583.871138:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941583.871144:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941583.871145:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941583.871148:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941583.871151:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941583.871154:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c3100 x1860112291494016/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941593 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941583.871162:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c3100 x1860112291494016/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941593 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941583.871177:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941583.871181:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941583.871183:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941583.871212:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941583.871217:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941583.871222:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941583.871224:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941583.871228:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941583.871234:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941583.871244:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941583.871249:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941583.871257:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c3100 x1860112291494016/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941593 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941583.871269:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c3100 x1860112291494016/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941593 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941583.871276:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941583.871278:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941583.871281:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941583.871284:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941583.871287:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941583.871289:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941583.871292:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941583.871299:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=84 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941583.871303:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941583.871306:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941583.871308:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941583.871312:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941583.871315:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941583.871318:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941583.871324:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c3100 x1860112291494016/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941593 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941583.871333:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941583.871336:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941583.871338:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941583.871341:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941583.871347:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941583.871350:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941583.871352:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941583.871356:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941583.871358:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941583.871361:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941583.871364:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941583.871368:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941583 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941583.871379:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941583.871387:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941583.871390:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941583.871393:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941583.871396:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941583.871398:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941583.871403:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941583.871406:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941583.871409:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941583.871414:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941583.871418:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941583.871421:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941583.871676:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941583.871681:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941583.871688:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c3100 x1860112291494016/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941593 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941583.871691:0:14790:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:2.0:1773941583.871700:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291494016:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00080000:3.0:1773941583.871706:0:14790:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00000001:2.0:1773941583.871707:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941583.871711:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00100000:3.0:1773941583.871712:0:14790:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000040:2.0:1773941583.871716:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c3100 x1860112291494016/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941593 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941583.871718:0:14790:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941583.871721:0:14790:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:3.0:1773941583.871728:0:14790:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:2.0:1773941583.871728:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941583.871733:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:3.0:1773941583.871734:0:14790:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000010:2.0:1773941583.871736:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03522800. 02000000:00000001:2.0:1773941583.871740:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:3.0:1773941583.871742:0:14790:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941583.871742:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941583.871745:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000001:00000001:3.0:1773941583.871747:0:14790:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:2.0:1773941583.871748:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:3.0:1773941583.871751:0:14790:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000010:2.0:1773941583.871752:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d1118b400. 00000001:00000001:3.0:1773941583.871757:0:14790:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941583.871757:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941583.871759:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 10000000:00000001:3.0:1773941583.871764:0:14790:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941583.871764:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c3100. 00000100:00000001:2.0:1773941583.871769:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941583.871770:0:14790:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941583.871771:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941583.871774:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1773941583.871776:0:14790:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941583.871777:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941583.871781:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941583.871785:0:14790:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941583.871785:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941583.871788:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941583.871791:0:14790:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941583.871795:0:14790:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1773941583.871799:0:14790:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:3.0:1773941583.871802:0:14790:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941583.871804:0:14790:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:3.0:1773941583.871807:0:14790:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:3.0:1773941583.871827:0:14790:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:3.0:1773941583.871830:0:14790:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941583.871840:0:14790:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:3.0:1773941583.871846:0:14790:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:3.0:1773941583.871851:0:14790:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941583.871855:0:14790:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941583.871857:0:14790:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941583.871860:0:14790:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941583.871861:0:14790:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941583.875013:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d04dd4e00. 00010000:00000010:1.1:1773941583.875023:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d01488800. 00000020:00000001:2.0:1773941584.891134:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941584.891140:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941584.891144:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1773941585.147096:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294975465, at 4294975488 00000100:00000010:0.0:1773941585.147113:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3720. 00000100:00000040:1.0:1773941585.147216:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294975465, at 4294975488 00000100:00000010:1.0:1773941585.147223:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f37e0. 10000000:00000001:1.0:1773941585.211075:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941585.211084:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941585.211090:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941585.211107:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941585.211109:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941585.211110:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941585.211113:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941585.211115:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941585.211118:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:1.0:1773941585.211121:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941585.211139:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941585.211147:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c0700. 00000020:00000040:1.0:1773941585.211152:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941585.211157:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941585.211160:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941585.211166:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941585.211174:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d181fc400. 00000100:00000001:1.0:1773941585.211181:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941585.211186:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941585.211190:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941585.211197:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d04dd4e00. 00010000:00000001:1.0:1773941585.211203:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941585.211206:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d01488800. 00000020:00000001:1.0:1773941585.211211:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941585.211213:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d01488800 with handle 0x8e86767622e2a63 to hash 00000020:00000001:1.0:1773941585.211216:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941585.211217:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941585.211220:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941585.211224:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941585.211227:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941585.211232:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a63 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941585.211242:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a63 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941585.211254:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a63 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941585.211261:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941585.211263:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941585.211271:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941585.211274:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330528610752 : -113743180940864 : ffff988d1a1ab9c0) 00000100:00000001:1.0:1773941585.211277:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941585.211285:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00000100:00000001:1.0:1773941585.211290:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941585.211294:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941585.211295:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941585.211297:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941585.211306:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c0700 x1860112291494144/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941585.211313:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941585.211315:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941585.211318:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941585.211320:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941585.211322:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941585.211325:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291494144:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941585.211330:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941585.211335:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941585.211337:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941585.211340:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941585.211345:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d11189400. 02000000:00000001:1.0:1773941585.211347:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941585.211354:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941585.211359:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c5d8. 00000400:00000010:1.0:1773941585.211366:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000100:00000200:1.0:1773941585.211373:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291494144, portal 25 00000100:00000001:1.0:1773941585.211375:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941585.211377:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00000100:00000040:1.0:1773941585.211383:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c0700 x1860112291494144/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941601 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941585.211391:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941585.211394:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941585.211397:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de990. 00000100:00000200:1.0:1773941585.211400:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291494144, offset 0 00000400:00000200:1.0:1773941585.211407:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941585.211414:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941585.211425:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941585.211456:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941585.211460:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941585.211464:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941585.211467:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941585.211470:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[138, 137] 00000400:00000200:1.0:1773941585.211474:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:1.0:1773941585.211478:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [249:249:256:4294967295] 192.168.204.124@tcp LPNI seq info [139:139:8:4294967295] 00000400:00000200:1.0:1773941585.211482:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941585.211492:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:1.0:1773941585.211498:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:1.0:1773941585.211503:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfda00. 00000800:00000200:1.0:1773941585.211509:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:1.0:1773941585.211516:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:1.0:1773941585.211521:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfda00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941585.211537:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941585.211540:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941585.211543:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294976807 00000100:00000010:1.0:1773941585.211547:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f37e0. 00000100:00000001:1.0:1773941585.211554:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1773941585.211562:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294976807 00000100:00000010:1.0:1773941585.211565:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f39c0. 00000100:00000001:1.0:1773941585.211568:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1773941585.211573:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941585.211576:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941585.211593:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941585.211596:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941585.211599:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 11 seconds 00000100:00000001:1.0:1773941585.211601:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941585.211604:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941585.211697:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941585.211703:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfda00. 00000400:00000200:0.0:1773941585.211707:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941585.211711:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941585.211716:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de990 00000400:00000010:0.0:1773941585.211718:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de990. 00000100:00000001:0.0:1773941585.211721:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941585.211725:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c0700 x1860112291494144/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941601 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941585.211743:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941585.211746:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d203c0700 x1860112291494144/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941601 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941585.211751:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941585.211753:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941585.213178:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941585.213190:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941585.213192:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941585.213193:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941585.213197:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941585.213201:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941585.213205:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941585.213208:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941585.213211:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941585.213215:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324607900 00000400:00000200:0.0:1773941585.213218:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941585.213221:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0x1091 [1] + 224 00000800:00000001:0.0:1773941585.213226:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941585.213232:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941585.213234:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941585.213236:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941585.213239:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941585.213243:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c0700 x1860112291494144/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941601 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941585.213250:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d203c0700 x1860112291494144/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941601 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941585.213263:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941585.213277:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941585.213283:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941585.213291:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941585.213300:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941585.213302:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941585.213303:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:1.0:1773941585.213306:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:1.0:1773941585.213308:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:1.0:1773941585.213309:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941585.213313:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c0700 x1860112291494144/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941601 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941585.213320:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c0700 x1860112291494144/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941601 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941585.213325:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941585.213327:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941585.213329:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941585.213330:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941585.213332:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941585.213334:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941585.213336:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941585.213341:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=86 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941585.213343:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941585.213345:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941585.213348:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:1.0:1773941585.213351:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941585.213353:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941585.213355:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941585.213358:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941585.213361:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941585.213363:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941585.213365:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941585.213367:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941585.213368:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941585.213371:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d203c0700 x1860112291494144/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941601 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941585.213383:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941585.213384:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941585.213385:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941585.213387:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941585.213389:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941585.213390:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941585.213392:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941585.213393:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941585.213393:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941585.213394:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941585.213395:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941585.213397:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941585.213399:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941585.213400:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941585.213402:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 306 00000400:00000200:1.0:1773941585.213406:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941585.213409:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941585.213411:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941585.213414:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941585.213415:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941585.213417:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941585.213419:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941585.213420:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941585.213422:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941585.213424:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941585.213425:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941585.213427:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941585.213437:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941585.213438:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941585.213439:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941585.213441:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941585.213443:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941585.213459:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941585.213461:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941585.213463:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941585.213465:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941585.213467:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941585.213469:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941585.213470:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941585.213472:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c1c00. 00000020:00000040:1.0:1773941585.213474:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941585.213476:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941585.213477:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941585.213479:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941585.213480:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941585.213482:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941585.213485:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d1118b800. 00000100:00000001:1.0:1773941585.213487:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941585.213491:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c1c00 x1860112291494272/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941585.213497:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941585.213499:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941585.213502:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000033e82992] to pc [ptlrpcd_rcv+-1] req@ffff988d203c1c00 x1860112291494272/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941585.213516:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941585.213518:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941585.213519:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941585.213521:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941585.213523:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941585.213524:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941585.213525:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941585.213526:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941585.213528:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941585.213531:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c0700 x1860112291494144/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941601 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941585.213543:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941585.213546:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941585.213547:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941585.213549:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941585.213553:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c0700 x1860112291494144/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941601 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941585.213559:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291494144:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941585.213562:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941585.213564:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941585.213565:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941585.213567:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941585.213570:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941585.213571:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1773941585.213572:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c0700 x1860112291494144/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941601 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941585.213589:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941585.213591:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941585.213593:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941585.213596:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941585.213598:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00010000:00000001:1.0:1773941585.213599:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941585.213603:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941585.213605:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941585.213608:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488800 refcount=4 00000100:00000040:2.0:1773941585.213609:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c1c00 x1860112291494272/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1773941585.213610:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941585.213614:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00010000:1.0:1773941585.213619:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a63 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941585.213632:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:1.0:1773941585.213634:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941585.213636:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941585.213638:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a63 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941585.213641:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941585.213644:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941585.213649:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941585.213657:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291494272:192.168.204.114@tcp:250:ll_cfg_requeue.0 00010000:00000001:1.0:1773941585.213657:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941585.213661:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a63 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941585.213663:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941585.213667:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00000001:1.0:1773941585.213669:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 02000000:00000001:2.0:1773941585.213671:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941585.213672:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941585.213675:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a63 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941585.213676:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:1.0:1773941585.213682:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 02000000:00000010:2.0:1773941585.213683:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a7400. 00010000:00000001:1.0:1773941585.213684:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941585.213690:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941585.213690:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941585.213693:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a63 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941585.213699:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:1.0:1773941585.213703:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d01488800 already unused, calling callback (ffffffffc12b4c20) 00000400:00000010:2.0:1773941585.213705:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 10000000:00000001:1.0:1773941585.213706:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941585.213710:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a63 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941585.213713:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00010000:00000001:1.0:1773941585.213716:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941585.213719:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000200:2.0:1773941585.213720:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291494272, portal 25 00000020:00000001:1.0:1773941585.213721:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941585.213723:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488800 refcount=5 00000100:00000001:2.0:1773941585.213725:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000020:00000001:1.0:1773941585.213727:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00000100:00000001:2.0:1773941585.213729:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631461888 : -113743078089728 : ffff988d203c1c00) 00010000:00000001:1.0:1773941585.213730:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941585.213736:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941585.213739:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a63 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941585.213741:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c1c00 x1860112291494272/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941595 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000001:1.0:1773941585.213751:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000001:2.0:1773941585.213756:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 10000000:00010000:1.0:1773941585.213758:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a63 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941585.213764:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 10000000:01000000:1.0:1773941585.213768:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000010:2.0:1773941585.213770:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516aa0. 10000000:00000040:1.0:1773941585.213771:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941585.213773:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1773941585.213777:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291494272, offset 0 00010000:00010000:1.0:1773941585.213777:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941585.213780:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941585.213784:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941585.213785:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000400:00000200:2.0:1773941585.213788:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00000001:1.0:1773941585.213789:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941585.213792:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941585.213795:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d01488800 with handle 0x8e86767622e2a63 from hash 00000400:00000200:2.0:1773941585.213799:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941585.213799:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941585.213802:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941585.213804:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941585.213805:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941585.213808:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000400:00000200:2.0:1773941585.213809:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:1.0:1773941585.213810:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941585.213814:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:2.0:1773941585.213832:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:1.0:1773941585.213833:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941585.213836:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941585.213839:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941585.213842:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00010000:1.0:1773941585.213843:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a63 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941585.213850:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941585.213852:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941585.213854:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941585.213855:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941585.213856:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941585.213858:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941585.213859:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941585.213861:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941585.213866:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a63 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941585.213868:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941585.213880:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[138, 139] 00010000:00000040:1.0:1773941585.213882:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d04dd4e00 count: 0 00000400:00000200:2.0:1773941585.213887:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:2.0:1773941585.213895:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [250:250:256:4294967295] 192.168.204.114@tcp LPNI seq info [140:140:8:4294967295] 00000400:00000200:2.0:1773941585.213904:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941585.213904:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941585.213909:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941585.213913:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941585.213922:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c0700 x1860112291494144/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941601 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1773941585.213930:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000100:00000001:1.0:1773941585.213939:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941585.213943:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000800:00000200:2.0:1773941585.213949:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 02000000:00000010:1.0:1773941585.213949:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d11189400. 02000000:00000001:1.0:1773941585.213955:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941585.213958:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000800:00000010:2.0:1773941585.213960:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0db600. 00000020:00000040:1.0:1773941585.213961:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941585.213968:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941585.213971:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d181fc400. 02000000:00000001:1.0:1773941585.214028:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941585.214030:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941585.214033:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c0700. 00000800:00000200:2.0:1773941585.214037:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000100:00000001:1.0:1773941585.214037:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941585.214040:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941585.214044:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941585.214047:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941585.214050:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000800:00000200:2.0:1773941585.214053:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 10000000:00000010:1.0:1773941585.214055:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d04d675c0. 00000800:00000200:2.0:1773941585.214061:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0db600 type 1, nob 616 niov 1 nkiov 1 00000020:00000010:1.0:1773941585.214062:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d181fc400. 00000020:00000010:1.0:1773941585.214070:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d181fd000. 00000040:00000010:1.0:1773941585.214075:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d181fce00. 00000020:00000010:1.0:1773941585.214080:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d18b18000. 00000100:00000001:2.0:1773941585.214082:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941585.214085:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d181fc800. 00000400:00000200:2.0:1773941585.214108:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 80000000:00000010:1.0:1773941585.214108:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d054f3d20. 00000100:00000001:2.0:1773941585.214115:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294976810 00040000:00000010:1.0:1773941585.214115:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d181fc200. 20000000:00000010:1.0:1773941585.214122:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d181fd800. 00000100:00000001:2.0:1773941585.214125:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294976807 10000000:00000040:1.0:1773941585.214128:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:1.0:1773941585.214132:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941585.214133:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294976810 00000020:00000010:1.0:1773941585.214137:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d181fd000. 00000100:00000001:2.0:1773941585.214140:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294976807 00000040:00000010:1.0:1773941585.214143:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d181fce00. 00000100:00000001:2.0:1773941585.214146:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941585.214150:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941585.214154:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d18b18000. 00000100:00000001:2.0:1773941585.214156:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941585.214158:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1773941585.214161:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1773941585.214162:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d181fc800. 80000000:00000010:1.0:1773941585.214177:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d054f3d20. 00000100:00000001:2.0:1773941585.214181:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000800:00000010:0.0:1773941585.214181:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0db600. 00040000:00000010:1.0:1773941585.214184:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d181fc200. 00000100:00000001:2.0:1773941585.214186:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000400:00000200:0.0:1773941585.214188:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 20000000:00000010:1.0:1773941585.214191:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d181fd800. 00000100:00000001:2.0:1773941585.214192:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:0.0:1773941585.214196:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:2.0:1773941585.214197:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000020:00000010:1.0:1773941585.214198:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d181fc400. 00000400:00000200:0.0:1773941585.214201:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516aa0 00000100:00000001:2.0:1773941585.214202:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 10000000:00000010:1.0:1773941585.214204:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d04d675c0. 00000400:00000010:0.0:1773941585.214205:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516aa0. 00000100:00000001:2.0:1773941585.214207:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:01000000:1.0:1773941585.214212:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 00000100:00000001:0.0:1773941585.214213:0:2393:0:(events.c:34:request_out_callback()) Process entered 10000000:00000001:1.0:1773941585.214219:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000200:0.0:1773941585.214220:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c1c00 x1860112291494272/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941595 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00020000:1.0:1773941585.214225:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941585.214230:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 00000100:00000001:0.0:1773941585.214233:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 10000000:00000001:1.0:1773941585.214234:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941585.214238:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 00000100:00000040:0.0:1773941585.214238:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c1c00 x1860112291494272/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941595 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000001:1.0:1773941585.214244:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000100:00000001:0.0:1773941585.214249:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941585.214251:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941585.215751:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941585.215759:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941585.215761:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941585.215762:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941585.215765:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941585.215769:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941585.215773:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941585.215776:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941585.215778:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941585.215780:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324607980 00000400:00000200:0.0:1773941585.215783:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941585.215786:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0x10a1 [1] + 0 00000800:00000001:0.0:1773941585.215790:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941585.215795:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941585.215796:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941585.215799:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941585.215802:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941585.215805:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c1c00 x1860112291494272/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941595 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941585.215813:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c1c00 x1860112291494272/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941595 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941585.215834:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941585.215839:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941585.215842:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941585.215861:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941585.215864:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941585.215867:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941585.215868:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941585.215870:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941585.215878:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941585.215879:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941585.215886:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941585.215890:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c1c00 x1860112291494272/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941595 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941585.215897:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c1c00 x1860112291494272/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941595 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941585.215903:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941585.215904:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941585.215905:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941585.215906:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941585.215907:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941585.215909:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941585.215910:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941585.215912:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=86 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941585.215915:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941585.215917:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941585.215918:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941585.215920:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941585.215921:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941585.215923:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941585.215926:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c1c00 x1860112291494272/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941595 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941585.215931:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941585.215932:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941585.215934:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941585.215935:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941585.215939:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941585.215940:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941585.215941:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941585.215942:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941585.215944:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941585.215946:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941585.215948:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:2.0:1773941585.215954:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941585 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941585.215956:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941585.215982:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:2.0:1773941585.215985:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941585.215987:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941585.215989:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941585.215990:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941585.215994:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:2.0:1773941585.215996:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941585.215998:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941585.216001:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941585.216004:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:2.0:1773941585.216005:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941585.216210:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941585.216215:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941585.216221:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c1c00 x1860112291494272/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941595 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941585.216229:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c1c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291494272:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941585.216234:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941585.216236:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941585.216239:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c1c00 x1860112291494272/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941595 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941585.216241:0:14792:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:3.0:1773941585.216247:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941585.216249:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00080000:2.0:1773941585.216251:0:14792:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 02000000:00000010:3.0:1773941585.216253:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a7400. 00000100:00100000:2.0:1773941585.216257:0:14792:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000001:3.0:1773941585.216258:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941585.216261:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:2.0:1773941585.216261:0:14792:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941585.216264:0:14792:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000020:00000040:3.0:1773941585.216265:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941585.216269:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:2.0:1773941585.216269:0:14792:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000010:3.0:1773941585.216273:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d1118b800. 00000001:00000001:2.0:1773941585.216274:0:14792:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:3.0:1773941585.216276:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000001:00000001:2.0:1773941585.216278:0:14792:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941585.216279:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941585.216282:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c1c00. 00000001:00000001:2.0:1773941585.216283:0:14792:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941585.216285:0:14792:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:3.0:1773941585.216286:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000001:00000001:2.0:1773941585.216288:0:14792:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941585.216289:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:2.0:1773941585.216291:0:14792:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941585.216292:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941585.216296:0:14792:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941585.216298:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941585.216301:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1773941585.216301:0:14792:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:3.0:1773941585.216307:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941585.216308:0:14792:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941585.216310:0:14792:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:2.0:1773941585.216311:0:14792:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1773941585.216313:0:14792:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:3.0:1773941585.216316:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941585.216317:0:14792:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941585.216319:0:14792:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:2.0:1773941585.216322:0:14792:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:2.0:1773941585.216339:0:14792:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:2.0:1773941585.216340:0:14792:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941585.216342:0:14792:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:2.0:1773941585.216344:0:14792:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:2.0:1773941585.216346:0:14792:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941585.216348:0:14792:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941585.216349:0:14792:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941585.216351:0:14792:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941585.216352:0:14792:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941585.219026:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d04dd4e00. 00010000:00000010:1.1:1773941585.219037:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d01488800. 00000020:00000001:2.0:1773941585.915130:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941585.915137:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941585.915140:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941586.491083:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294976810, at 4294976832 00000100:00000040:0.0:1773941586.491083:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294976810, at 4294976832 00000100:00000010:1.0:1773941586.491116:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f39c0. 00000100:00000010:0.0:1773941586.491118:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f37e0. 00000020:00000001:2.0:1773941586.939130:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941586.939138:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941586.939143:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941587.131073:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941587.131082:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941587.131087:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941587.131090:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941587.131092:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941587.131093:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941587.131096:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941587.131098:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941587.131100:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:1.0:1773941587.131104:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941587.131106:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941587.131118:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c0700. 00000020:00000040:1.0:1773941587.131123:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941587.131128:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941587.131132:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941587.131137:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941587.131145:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d181fd800. 00000100:00000001:1.0:1773941587.131152:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941587.131158:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941587.131175:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941587.131183:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d04dd4e00. 00010000:00000001:1.0:1773941587.131189:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941587.131192:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d01488800. 00000020:00000001:1.0:1773941587.131198:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941587.131201:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d01488800 with handle 0x8e86767622e2a6a to hash 00000020:00000001:1.0:1773941587.131204:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941587.131205:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941587.131209:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941587.131214:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941587.131218:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941587.131222:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a6a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941587.131234:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a6a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941587.131248:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a6a lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941587.131256:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941587.131258:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941587.131265:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941587.131268:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330528610752 : -113743180940864 : ffff988d1a1ab9c0) 00000100:00000001:1.0:1773941587.131271:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941587.131272:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00000100:00000001:1.0:1773941587.131278:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941587.131282:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941587.131285:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941587.131287:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941587.131297:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c0700 x1860112291494400/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941587.131308:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941587.131316:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941587.131320:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941587.131323:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941587.131326:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941587.131330:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291494400:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941587.131335:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941587.131340:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941587.131343:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941587.131345:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941587.131350:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d11189400. 02000000:00000001:1.0:1773941587.131352:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941587.131360:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941587.131365:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c5d8. 00000400:00000010:1.0:1773941587.131372:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000100:00000200:1.0:1773941587.131380:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291494400, portal 25 00000100:00000001:1.0:1773941587.131382:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941587.131384:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00000100:00000040:1.0:1773941587.131390:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c0700 x1860112291494400/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941603 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941587.131400:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941587.131403:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941587.131406:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113debb0. 00000100:00000200:1.0:1773941587.131409:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291494400, offset 0 00000400:00000200:1.0:1773941587.131416:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941587.131425:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941587.131432:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941587.131439:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941587.131443:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941587.131447:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941587.131450:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941587.131454:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[140, 139] 00000400:00000200:1.0:1773941587.131459:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:1.0:1773941587.131463:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [251:251:256:4294967295] 192.168.204.124@tcp LPNI seq info [141:141:8:4294967295] 00000400:00000200:1.0:1773941587.131473:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941587.131485:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:1.0:1773941587.131492:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:1.0:1773941587.131497:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfd800. 00000800:00000200:1.0:1773941587.131504:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:1.0:1773941587.131512:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:1.0:1773941587.131517:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfd800 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941587.131536:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941587.131557:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941587.131561:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294978727 00000100:00000010:1.0:1773941587.131566:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f39c0. 00000100:00000001:1.0:1773941587.131573:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1773941587.131580:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294978727 00000100:00000010:1.0:1773941587.131584:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3d20. 00000100:00000001:1.0:1773941587.131588:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1773941587.131591:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941587.131595:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941587.131597:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941587.131600:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941587.131603:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 11 seconds 00000100:00000001:1.0:1773941587.131606:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941587.131609:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941587.131676:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941587.131681:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfd800. 00000400:00000200:0.0:1773941587.131685:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941587.131690:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941587.131694:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113debb0 00000400:00000010:0.0:1773941587.131696:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113debb0. 00000100:00000001:0.0:1773941587.131699:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941587.131703:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c0700 x1860112291494400/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941603 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941587.131712:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941587.131714:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d203c0700 x1860112291494400/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941603 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941587.131720:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941587.131721:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941587.133059:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941587.133071:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941587.133073:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941587.133074:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941587.133078:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941587.133082:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941587.133086:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941587.133089:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941587.133092:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941587.133096:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324607a00 00000400:00000200:0.0:1773941587.133099:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941587.133102:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0x10a9 [1] + 224 00000800:00000001:0.0:1773941587.133114:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941587.133120:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941587.133122:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941587.133125:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941587.133128:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941587.133131:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c0700 x1860112291494400/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941603 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941587.133138:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d203c0700 x1860112291494400/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941603 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941587.133152:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941587.133159:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941587.133161:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941587.133215:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941587.133221:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941587.133223:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941587.133225:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:1.0:1773941587.133229:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:1.0:1773941587.133230:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:1.0:1773941587.133232:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941587.133237:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c0700 x1860112291494400/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941603 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941587.133245:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c0700 x1860112291494400/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941603 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941587.133256:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941587.133258:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941587.133260:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941587.133262:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941587.133264:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941587.133266:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941587.133268:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941587.133273:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=88 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941587.133276:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941587.133278:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941587.133281:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:1.0:1773941587.133283:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941587.133285:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941587.133287:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941587.133291:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941587.133294:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941587.133296:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941587.133298:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941587.133300:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941587.133302:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941587.133304:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d203c0700 x1860112291494400/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941603 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941587.133310:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941587.133310:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941587.133312:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941587.133314:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941587.133316:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941587.133317:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941587.133318:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941587.133319:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941587.133320:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941587.133321:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941587.133325:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941587.133327:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941587.133328:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941587.133330:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941587.133332:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 308 00000400:00000200:1.0:1773941587.133336:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941587.133339:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941587.133342:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941587.133344:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941587.133346:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941587.133347:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941587.133350:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941587.133351:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941587.133353:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941587.133355:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941587.133356:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941587.133358:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941587.133359:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941587.133361:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941587.133362:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941587.133363:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941587.133365:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941587.133367:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941587.133369:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941587.133370:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941587.133372:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941587.133374:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941587.133375:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941587.133377:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941587.133379:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c1180. 00000020:00000040:1.0:1773941587.133381:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941587.133383:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941587.133389:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941587.133391:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941587.133393:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941587.133395:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941587.133399:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d1118b800. 00000100:00000001:1.0:1773941587.133402:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941587.133406:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c1180 x1860112291494528/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941587.133412:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941587.133414:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941587.133417:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000223b0731] to pc [ptlrpcd_rcv+-1] req@ffff988d203c1180 x1860112291494528/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941587.133433:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941587.133435:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941587.133436:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941587.133438:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941587.133444:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941587.133445:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941587.133447:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941587.133448:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941587.133450:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941587.133454:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c0700 x1860112291494400/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941603 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941587.133460:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941587.133462:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941587.133464:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941587.133466:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941587.133466:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941587.133471:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1773941587.133471:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c0700 x1860112291494400/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941603 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941587.133475:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941587.133482:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c1180 x1860112291494528/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941587.133483:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291494400:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941587.133493:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941587.133495:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941587.133497:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941587.133503:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:3.0:1773941587.133504:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941587.133507:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941587.133508:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 02000000:00000001:3.0:1773941587.133511:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941587.133512:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00100000:3.0:1773941587.133517:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291494528:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000040:1.0:1773941587.133519:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c0700 x1860112291494400/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941603 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941587.133524:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941587.133528:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941587.133530:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941587.133532:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941587.133533:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:1.0:1773941587.133535:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1a1ab9c0. 02000000:00000010:3.0:1773941587.133539:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03520c00. 00000100:00000001:1.0:1773941587.133539:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941587.133555:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:1.0:1773941587.133559:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 02000000:00000001:3.0:1773941587.133561:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941587.133563:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941587.133565:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941587.133567:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488800 refcount=4 00000020:00000001:1.0:1773941587.133569:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941587.133571:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00000400:00000200:3.0:1773941587.133573:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:1.0:1773941587.133575:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a6a lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:3.0:1773941587.133579:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00010000:00000001:1.0:1773941587.133584:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000010:3.0:1773941587.133588:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00010000:00010000:1.0:1773941587.133589:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a6a lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:3.0:1773941587.133596:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291494528, portal 25 00010000:00000001:1.0:1773941587.133597:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941587.133600:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a6a lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941587.133601:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941587.133605:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631459200 : -113743078092416 : ffff988d203c1180) 00010000:00000001:1.0:1773941587.133609:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941587.133611:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941587.133614:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a6a lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:3.0:1773941587.133617:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c1180 x1860112291494528/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941597 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941587.133620:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941587.133627:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941587.133630:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:1.0:1773941587.133632:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000100:00000040:3.0:1773941587.133637:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00010000:1.0:1773941587.133638:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a6a lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:3.0:1773941587.133643:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac2a8. 00000100:00000200:3.0:1773941587.133650:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291494528, offset 0 00010000:00010000:1.0:1773941587.133651:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d01488800 already unused, calling callback (ffffffffc12b4c20) 10000000:00000001:1.0:1773941587.133657:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:3.0:1773941587.133660:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 10000000:00010000:1.0:1773941587.133662:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a6a lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941587.133669:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941587.133676:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000400:00000200:3.0:1773941587.133677:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:1.0:1773941587.133680:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941587.133682:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941587.133684:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488800 refcount=5 00000020:00000001:1.0:1773941587.133691:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00000400:00000200:3.0:1773941587.133692:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:1.0:1773941587.133696:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00000400:00000200:3.0:1773941587.133697:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941587.133702:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941587.133703:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:3.0:1773941587.133705:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941587.133709:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[140, 141] 00010000:00010000:1.0:1773941587.133709:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a6a lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941587.133720:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941587.133727:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [252:252:256:4294967295] 192.168.204.114@tcp LPNI seq info [142:142:8:4294967295] 10000000:00000001:1.0:1773941587.133728:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:3.0:1773941587.133732:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 10000000:00010000:1.0:1773941587.133732:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a6a lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1773941587.133742:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:3.0:1773941587.133743:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 10000000:00000040:1.0:1773941587.133747:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941587.133750:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941587.133751:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00010000:00010000:1.0:1773941587.133754:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000800:00000010:3.0:1773941587.133756:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74c00. 00010000:00000001:1.0:1773941587.133757:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941587.133761:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941587.133764:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000800:00000200:3.0:1773941587.133766:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00010000:00000001:1.0:1773941587.133767:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941587.133770:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941587.133773:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d01488800 with handle 0x8e86767622e2a6a from hash 00000800:00000200:3.0:1773941587.133778:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00010000:00000001:1.0:1773941587.133782:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000800:00000200:3.0:1773941587.133784:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74c00 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941587.133785:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941587.133788:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941587.133790:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941587.133793:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941587.133794:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941587.133796:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941587.133796:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941587.133797:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941587.133799:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941587.133801:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a6a lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941587.133805:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941587.133806:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941587.133807:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941587.133809:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941587.133812:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941587.133814:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:3.0:1773941587.133816:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941587.133816:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941587.133819:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941587.133822:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294978729 00010000:00010000:1.0:1773941587.133823:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a6a lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941587.133832:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294978727 00010000:00000040:1.0:1773941587.133832:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d04dd4e00 count: 0 00000100:00000001:3.0:1773941587.133839:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294978729 00000100:00000001:3.0:1773941587.133842:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294978727 00010000:00000001:1.0:1773941587.133844:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:3.0:1773941587.133846:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941587.133847:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941587.133849:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941587.133851:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1773941587.133853:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941587.133856:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941587.133857:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c0700 x1860112291494400/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941603 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941587.133863:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941587.133864:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1773941587.133867:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 02000000:00000001:1.0:1773941587.133869:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1773941587.133872:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000010:1.0:1773941587.133873:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d11189400. 00000100:00000001:3.0:1773941587.133874:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1773941587.133877:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000800:00000200:0.0:1773941587.133877:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1773941587.133878:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941587.133881:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:3.0:1773941587.133884:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941587.133884:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000800:00000010:0.0:1773941587.133884:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74c00. 00000020:00000001:1.0:1773941587.133891:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 00000400:00000200:0.0:1773941587.133892:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000010:1.0:1773941587.133898:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d181fd800. 00000400:00000200:0.0:1773941587.133900:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 02000000:00000001:1.0:1773941587.133903:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941587.133905:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000400:00000200:0.0:1773941587.133905:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac2a8 00000100:00000010:1.0:1773941587.133909:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c0700. 00000400:00000010:0.0:1773941587.133910:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac2a8. 00000100:00000001:1.0:1773941587.133913:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941587.133915:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1773941587.133916:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941587.133919:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941587.133922:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000100:00000200:0.0:1773941587.133924:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c1180 x1860112291494528/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941597 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000001:1.0:1773941587.133925:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941587.133930:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d04d67500. 00000020:00000010:1.0:1773941587.133936:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d181fd800. 00000100:00000001:0.0:1773941587.133937:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000020:00000010:1.0:1773941587.133942:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d181fc200. 00000100:00000040:0.0:1773941587.133942:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c1180 x1860112291494528/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941597 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000040:00000010:1.0:1773941587.133948:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d181fc800. 00000100:00000001:0.0:1773941587.133953:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941587.133955:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d18b18000. 00000100:00000001:0.0:1773941587.133956:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:1.0:1773941587.133990:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d181fce00. 80000000:00000010:1.0:1773941587.133994:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d054f3b40. 00040000:00000010:1.0:1773941587.133998:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d181fd000. 20000000:00000010:1.0:1773941587.134001:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d181fd600. 10000000:00000040:1.0:1773941587.134005:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:1.0:1773941587.134007:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941587.134011:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d181fc200. 00000040:00000010:1.0:1773941587.134014:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d181fc800. 00000020:00000010:1.0:1773941587.134017:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941587.134021:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d181fce00. 80000000:00000010:1.0:1773941587.134024:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d054f3b40. 00040000:00000010:1.0:1773941587.134027:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d181fd000. 20000000:00000010:1.0:1773941587.134030:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d181fd600. 00000020:00000010:1.0:1773941587.134033:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d181fd800. 10000000:00000010:1.0:1773941587.134036:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d04d67500. 10000000:01000000:1.0:1773941587.134039:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941587.134043:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941587.134046:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941587.134048:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941587.134050:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941587.134057:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941587.134059:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941587.135156:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941587.135162:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941587.135164:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941587.135165:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941587.135169:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941587.135172:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941587.135176:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941587.135184:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941587.135187:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941587.135189:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324607a80 00000400:00000200:0.0:1773941587.135192:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941587.135194:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0x10b9 [1] + 0 00000800:00000001:0.0:1773941587.135198:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941587.135202:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941587.135204:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941587.135207:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941587.135211:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941587.135217:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c1180 x1860112291494528/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941597 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941587.135228:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c1180 x1860112291494528/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941597 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941587.135249:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941587.135255:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941587.135259:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941587.135275:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941587.135283:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941587.135286:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941587.135287:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941587.135289:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941587.135293:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941587.135294:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941587.135297:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941587.135303:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c1180 x1860112291494528/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941597 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941587.135314:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c1180 x1860112291494528/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941597 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941587.135322:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941587.135323:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941587.135326:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941587.135327:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941587.135329:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941587.135331:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941587.135333:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941587.135337:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=88 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941587.135341:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941587.135343:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941587.135345:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941587.135348:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941587.135350:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941587.135353:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941587.135357:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c1180 x1860112291494528/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941597 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941587.135362:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941587.135364:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941587.135365:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941587.135367:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941587.135370:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941587.135372:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941587.135373:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941587.135374:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941587.135381:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941587.135383:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941587.135384:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941587.135387:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941587 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941587.135389:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941587.135394:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941587.135396:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941587.135397:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941587.135398:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941587.135399:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941587.135402:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941587.135404:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941587.135406:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941587.135408:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941587.135411:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941587.135413:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941587.135653:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941587.135662:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941587.135667:0:14793:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000040:2.0:1773941587.135669:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c1180 x1860112291494528/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941597 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1773941587.135675:0:14793:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:3.0:1773941587.135681:0:14793:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00100000:2.0:1773941587.135684:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c1180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291494528:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941587.135692:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941587.135693:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941587.135696:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c1180 x1860112291494528/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941597 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941587.135701:0:14793:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941587.135703:0:14793:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000001:2.0:1773941587.135703:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941587.135707:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:3.0:1773941587.135708:0:14793:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000010:2.0:1773941587.135710:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03520c00. 00000001:00000001:3.0:1773941587.135713:0:14793:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941587.135714:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1773941587.135716:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941587.135717:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000001:00000001:3.0:1773941587.135718:0:14793:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941587.135719:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941587.135722:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d1118b800. 00000001:00000001:3.0:1773941587.135724:0:14793:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:2.0:1773941587.135726:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000001:00000001:3.0:1773941587.135727:0:14793:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941587.135730:0:14793:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941587.135732:0:14793:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941587.135734:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1773941587.135737:0:14793:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000010:2.0:1773941587.135737:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c1180. 00000100:00080000:3.0:1773941587.135743:0:14793:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941587.135744:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941587.135746:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941587.135750:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941587.135751:0:14793:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941587.135754:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941587.135755:0:14793:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:2.0:1773941587.135756:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941587.135760:0:14793:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941587.135760:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1773941587.135764:0:14793:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941587.135772:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941587.135774:0:14793:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941587.135777:0:14793:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:3.0:1773941587.135779:0:14793:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:3.0:1773941587.135802:0:14793:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:3.0:1773941587.135804:0:14793:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941587.135806:0:14793:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:3.0:1773941587.135810:0:14793:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:3.0:1773941587.135813:0:14793:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941587.135816:0:14793:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941587.135818:0:14793:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941587.135821:0:14793:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941587.135823:0:14793:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941587.139009:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d04dd4e00. 00010000:00000010:1.1:1773941587.139018:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d01488800. 00000100:00000040:0.0:1773941587.195054:0:8:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (315) 00000020:00000001:2.0:1773941587.963141:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941587.963146:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941587.963150:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1773941588.411125:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294978729, at 4294978752 00000100:00000040:1.0:1773941588.411136:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294978729, at 4294978752 00000100:00000010:0.0:1773941588.411154:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f39c0. 00000100:00000010:1.0:1773941588.411157:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3d20. 10000000:00000001:1.0:1773941588.731082:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941588.731093:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941588.731099:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941588.731103:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941588.731105:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941588.731107:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941588.731111:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941588.731113:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941588.731117:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:1.0:1773941588.731121:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941588.731123:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941588.731137:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c0700. 00000020:00000040:1.0:1773941588.731143:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941588.731150:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941588.731155:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941588.731161:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941588.731171:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d181fd600. 00000100:00000001:1.0:1773941588.731180:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941588.731186:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941588.731190:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941588.731199:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d04dd4e00. 00010000:00000001:1.0:1773941588.731207:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941588.731211:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d01488800. 00000020:00000001:1.0:1773941588.731216:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941588.731220:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d01488800 with handle 0x8e86767622e2a71 to hash 00000020:00000001:1.0:1773941588.731223:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941588.731225:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941588.731229:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941588.731234:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941588.731238:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941588.731244:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a71 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941588.731290:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a71 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941588.731307:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a71 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941588.731315:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941588.731318:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941588.731327:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941588.731329:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330528610752 : -113743180940864 : ffff988d1a1ab9c0) 00000100:00000001:1.0:1773941588.731333:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941588.731335:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00000100:00000001:1.0:1773941588.731341:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941588.731346:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941588.731349:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941588.731351:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941588.731364:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c0700 x1860112291494656/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941588.731376:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941588.731379:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941588.731384:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941588.731387:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941588.731391:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941588.731395:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291494656:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941588.731401:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941588.731407:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941588.731410:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941588.731413:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941588.731419:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d11189400. 02000000:00000001:1.0:1773941588.731422:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941588.731431:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941588.731442:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c5d8. 00000400:00000010:1.0:1773941588.731450:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000100:00000200:1.0:1773941588.731459:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291494656, portal 25 00000100:00000001:1.0:1773941588.731462:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941588.731464:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00000100:00000040:1.0:1773941588.731471:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c0700 x1860112291494656/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941604 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941588.731482:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941588.731485:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941588.731489:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113dee58. 00000100:00000200:1.0:1773941588.731492:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291494656, offset 0 00000400:00000200:1.0:1773941588.731500:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941588.731556:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941588.731568:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941588.731574:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941588.731580:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941588.731586:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941588.731589:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941588.731594:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[142, 141] 00000400:00000200:1.0:1773941588.731599:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:1.0:1773941588.731604:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [253:253:256:4294967295] 192.168.204.124@tcp LPNI seq info [143:143:8:4294967295] 00000400:00000200:1.0:1773941588.731609:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941588.731622:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:1.0:1773941588.731629:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:1.0:1773941588.731636:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfdf00. 00000800:00000200:1.0:1773941588.731644:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:1.0:1773941588.731654:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:1.0:1773941588.731660:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfdf00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941588.731685:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941588.731691:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941588.731700:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294980327 00000100:00000010:1.0:1773941588.731713:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3d20. 00000100:00000001:1.0:1773941588.731722:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1773941588.731731:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294980327 00000100:00000010:1.0:1773941588.731749:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3b40. 00000100:00000001:1.0:1773941588.731754:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1773941588.731759:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941588.731763:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941588.731766:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941588.731769:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941588.731773:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 11 seconds 00000100:00000001:1.0:1773941588.731777:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941588.731781:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941588.731885:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941588.731893:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfdf00. 00000400:00000200:0.0:1773941588.731901:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941588.731910:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941588.731917:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113dee58 00000400:00000010:0.0:1773941588.731920:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113dee58. 00000100:00000001:0.0:1773941588.731925:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941588.731933:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c0700 x1860112291494656/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941604 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941588.731947:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941588.731952:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d203c0700 x1860112291494656/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941604 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941588.732013:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941588.732017:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941588.734100:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941588.734128:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941588.734132:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941588.734135:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941588.734143:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941588.734150:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941588.734158:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941588.734163:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941588.734169:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941588.734176:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324607b00 00000400:00000200:0.0:1773941588.734181:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941588.734187:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0x10c1 [1] + 224 00000800:00000001:0.0:1773941588.734196:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941588.734207:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941588.734210:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941588.734216:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941588.734223:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941588.734231:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c0700 x1860112291494656/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941604 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941588.734246:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d203c0700 x1860112291494656/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941604 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941588.734270:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941588.734291:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941588.734296:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941588.734330:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941588.734352:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941588.734354:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941588.734358:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:1.0:1773941588.734365:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:1.0:1773941588.734367:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:1.0:1773941588.734371:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941588.734380:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c0700 x1860112291494656/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941604 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941588.734396:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c0700 x1860112291494656/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941604 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941588.734406:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941588.734418:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941588.734421:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941588.734425:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941588.734428:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941588.734431:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941588.734434:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941588.734442:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=89 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941588.734447:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941588.734451:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941588.734456:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:1.0:1773941588.734471:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941588.734476:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941588.734480:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941588.734486:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941588.734491:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941588.734496:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941588.734500:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941588.734502:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941588.734505:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941588.734510:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d203c0700 x1860112291494656/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941604 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941588.734623:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941588.734625:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941588.734627:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941588.734631:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941588.734635:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941588.734638:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941588.734640:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941588.734642:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941588.734644:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941588.734646:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941588.734648:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941588.734650:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941588.734653:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941588.734657:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941588.734660:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 310 00000400:00000200:1.0:1773941588.734667:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941588.734673:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941588.734678:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941588.734682:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941588.734686:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941588.734689:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941588.734705:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941588.734707:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941588.734710:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941588.734714:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941588.734717:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941588.734720:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941588.734724:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941588.734727:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941588.734729:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941588.734732:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941588.734736:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941588.734740:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941588.734743:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941588.734746:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941588.734750:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941588.734753:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941588.734756:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941588.734759:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941588.734763:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c2300. 00000020:00000040:1.0:1773941588.734766:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941588.734770:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941588.734772:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941588.734775:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941588.734777:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941588.734781:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941588.734786:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d1118b400. 00000100:00000001:1.0:1773941588.734792:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941588.734799:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c2300 x1860112291494784/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941588.734810:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941588.734814:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941588.734820:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000093c3ab5c] to pc [ptlrpcd_rcv+-1] req@ffff988d203c2300 x1860112291494784/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941588.734866:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941588.734870:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941588.734872:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941588.734876:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941588.734878:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941588.734881:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941588.734883:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941588.734886:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941588.734889:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941588.734895:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c0700 x1860112291494656/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941604 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941588.734905:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941588.734909:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941588.734913:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941588.734916:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941588.734922:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c0700 x1860112291494656/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941604 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941588.734933:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291494656:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941588.734947:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941588.734948:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941588.734958:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941588.734959:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941588.735011:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941588.735014:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941588.735025:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c0700 x1860112291494656/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941604 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941588.735035:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941588.735047:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1a1ab9c0. 00000100:00000001:2.0:1773941588.735051:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941588.735053:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941588.735059:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:1.0:1773941588.735064:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000040:2.0:1773941588.735069:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c2300 x1860112291494784/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941588.735080:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941588.735086:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1773941588.735090:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000040:1.0:1773941588.735092:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488800 refcount=4 00000100:00000001:2.0:1773941588.735096:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941588.735098:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 02000000:00000001:2.0:1773941588.735104:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00010000:00000001:1.0:1773941588.735106:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 02000000:00000001:2.0:1773941588.735111:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00010000:00010000:1.0:1773941588.735115:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a71 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941588.735118:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941588.735133:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291494784:192.168.204.114@tcp:250:ll_cfg_requeue.0 00010000:00000001:1.0:1773941588.735147:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941588.735148:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941588.735154:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941588.735159:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941588.735159:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a71 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941588.735167:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:1.0:1773941588.735178:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 02000000:00000010:2.0:1773941588.735179:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a4c00. 00010000:00010000:1.0:1773941588.735185:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a71 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941588.735186:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941588.735209:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941588.735216:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:2.0:1773941588.735217:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:1.0:1773941588.735225:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a71 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941588.735227:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000400:00000010:2.0:1773941588.735237:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00010000:00000001:1.0:1773941588.735241:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000200:2.0:1773941588.735246:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291494784, portal 25 00010000:00000001:1.0:1773941588.735247:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941588.735258:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00000001:1.0:1773941588.735259:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000100:00000001:2.0:1773941588.735276:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631463680 : -113743078087936 : ffff988d203c2300) 00010000:00010000:1.0:1773941588.735279:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a71 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941588.735288:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d01488800 already unused, calling callback (ffffffffc12b4c20) 10000000:00000001:1.0:1773941588.735297:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941588.735303:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a71 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941588.735306:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c2300 x1860112291494784/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941598 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941588.735349:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000100:00000001:2.0:1773941588.735351:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941588.735356:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:2.0:1773941588.735362:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516000. 00010000:00000001:1.0:1773941588.735366:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000200:2.0:1773941588.735371:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291494784, offset 0 00000020:00000001:1.0:1773941588.735372:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941588.735379:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488800 refcount=5 00000400:00000200:2.0:1773941588.735388:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000020:00000001:1.0:1773941588.735389:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941588.735397:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00000400:00000200:2.0:1773941588.735405:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941588.735408:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941588.735415:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a71 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941588.735418:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941588.735432:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 10000000:00000001:1.0:1773941588.735442:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:2.0:1773941588.735443:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 10000000:00010000:1.0:1773941588.735450:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a71 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941588.735451:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941588.735461:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941588.735474:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[142, 143] 10000000:01000000:1.0:1773941588.735474:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941588.735484:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000400:00000200:2.0:1773941588.735490:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 10000000:00000001:1.0:1773941588.735491:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941588.735502:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [254:254:256:4294967295] 192.168.204.114@tcp LPNI seq info [144:144:8:4294967295] 00010000:00010000:1.0:1773941588.735502:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941588.735509:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941588.735537:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941588.735541:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000400:00000200:2.0:1773941588.735543:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941588.735546:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941588.735553:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941588.735563:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d01488800 with handle 0x8e86767622e2a71 from hash 00000400:00000200:2.0:1773941588.735571:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00010000:00000001:1.0:1773941588.735572:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941588.735579:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941588.735584:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941588.735591:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000800:00000200:2.0:1773941588.735592:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00010000:00000001:1.0:1773941588.735597:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941588.735602:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000800:00000010:2.0:1773941588.735603:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0db600. 00010000:00000001:1.0:1773941588.735607:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941588.735612:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941588.735619:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:2.0:1773941588.735620:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 10000000:00000001:1.0:1773941588.735629:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941588.735636:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a71 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941588.735638:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:2.0:1773941588.735653:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0db600 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941588.735671:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941588.735674:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941588.735675:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941588.735678:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000100:00000001:2.0:1773941588.735681:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941588.735681:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941588.735686:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941588.735691:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:2.0:1773941588.735692:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:1.0:1773941588.735697:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a71 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941588.735699:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294980331 00000100:00000001:2.0:1773941588.735714:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294980327 00010000:00000040:1.0:1773941588.735717:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d04dd4e00 count: 0 00000100:00000001:2.0:1773941588.735722:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294980331 00000100:00000001:2.0:1773941588.735733:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294980327 00010000:00000001:1.0:1773941588.735739:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941588.735744:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941588.735750:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941588.735755:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941588.735763:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941588.735764:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c0700 x1860112291494656/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941604 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941588.735771:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941588.735782:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941588.735787:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941588.735792:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1773941588.735796:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 02000000:00000010:1.0:1773941588.735797:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d11189400. 00000100:00000001:2.0:1773941588.735802:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 02000000:00000001:1.0:1773941588.735807:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1773941588.735809:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1773941588.735813:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:2.0:1773941588.735814:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000020:00000040:1.0:1773941588.735820:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941588.735822:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941588.735827:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941588.735829:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941588.735845:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d181fd600. 02000000:00000001:1.0:1773941588.735852:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941588.735854:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941588.735857:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c0700. 00000100:00000001:1.0:1773941588.735861:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941588.735864:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941588.735868:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941588.735871:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000800:00000200:0.0:1773941588.735874:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 10000000:00000001:1.0:1773941588.735877:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000800:00000010:0.0:1773941588.735884:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0db600. 10000000:00000010:1.0:1773941588.735887:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d04d67e00. 00000400:00000200:0.0:1773941588.735894:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000010:1.0:1773941588.735897:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d181fd600. 00000400:00000200:0.0:1773941588.735908:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941588.735917:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516000 00000020:00000010:1.0:1773941588.735918:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d181fd000. 00000040:00000010:1.0:1773941588.735925:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d181fce00. 00000020:00000010:1.0:1773941588.735934:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d18b18000. 00000400:00000010:0.0:1773941588.735938:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516000. 00000020:00000010:1.0:1773941588.735943:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d181fc800. 00000100:00000001:0.0:1773941588.735947:0:2393:0:(events.c:34:request_out_callback()) Process entered 80000000:00000010:1.0:1773941588.735951:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d054f36c0. 00000100:00000200:0.0:1773941588.735995:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c2300 x1860112291494784/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941598 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941588.736010:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941588.736016:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c2300 x1860112291494784/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941598 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941588.736029:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941588.736032:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00040000:00000010:1.0:1773941588.736033:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d181fc200. 20000000:00000010:1.0:1773941588.736041:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d181fd800. 10000000:00000040:1.0:1773941588.736059:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:1.0:1773941588.736063:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941588.736071:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d181fd000. 00000040:00000010:1.0:1773941588.736078:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d181fce00. 00000020:00000010:1.0:1773941588.736084:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941588.736090:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d181fc800. 80000000:00000010:1.0:1773941588.736096:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d054f36c0. 00040000:00000010:1.0:1773941588.736102:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d181fc200. 20000000:00000010:1.0:1773941588.736106:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d181fd800. 00000020:00000010:1.0:1773941588.736110:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d181fd600. 10000000:00000010:1.0:1773941588.736113:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d04d67e00. 10000000:01000000:1.0:1773941588.736117:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941588.736121:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941588.736123:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941588.736136:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941588.736138:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941588.736140:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941588.736143:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941588.738288:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941588.738301:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941588.738306:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941588.738313:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941588.738322:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941588.738331:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941588.738343:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941588.738368:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941588.738377:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941588.738385:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324607b80 00000400:00000200:0.0:1773941588.738393:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941588.738401:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0x10d1 [1] + 0 00000800:00000001:0.0:1773941588.738410:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941588.738428:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941588.738431:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941588.738436:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941588.738441:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941588.738448:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c2300 x1860112291494784/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941598 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941588.738461:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c2300 x1860112291494784/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941598 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941588.738493:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941588.738538:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941588.738546:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941588.738589:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941588.738595:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941588.738601:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941588.738603:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941588.738606:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941588.738612:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941588.738615:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941588.738618:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941588.738627:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c2300 x1860112291494784/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941598 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941588.738641:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c2300 x1860112291494784/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941598 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941588.738650:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941588.738652:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941588.738655:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941588.738657:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941588.738659:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941588.738661:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941588.738664:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941588.738668:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=89 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941588.738672:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941588.738676:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941588.738677:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941588.738680:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941588.738683:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941588.738686:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941588.738702:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c2300 x1860112291494784/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941598 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941588.738711:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941588.738714:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941588.738716:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941588.738719:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941588.738725:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941588.738727:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941588.738729:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941588.738732:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941588.738735:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941588.738737:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941588.738740:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:2.0:1773941588.738744:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941588 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941588.738748:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941588.738755:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:2.0:1773941588.738757:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941588.738759:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941588.738762:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941588.738764:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941588.738768:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:2.0:1773941588.738771:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941588.738773:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941588.738777:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941588.738781:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:2.0:1773941588.738784:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941588.739157:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941588.739164:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941588.739175:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c2300 x1860112291494784/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941598 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941588.739190:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c2300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291494784:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941588.739197:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941588.739201:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941588.739204:0:14794:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000040:3.0:1773941588.739207:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c2300 x1860112291494784/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941598 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941588.739220:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00080000:2.0:1773941588.739221:0:14794:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 02000000:00000001:3.0:1773941588.739225:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00100000:2.0:1773941588.739231:0:14794:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000010:3.0:1773941588.739232:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a4c00. 00000100:00000001:2.0:1773941588.739240:0:14794:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:3.0:1773941588.739242:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 10000000:01000000:2.0:1773941588.739245:0:14794:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000020:00000001:3.0:1773941588.739247:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:2.0:1773941588.739252:0:14794:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000040:3.0:1773941588.739253:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941588.739257:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941588.739259:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d1118b400. 02000000:00000001:3.0:1773941588.739263:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000001:00000001:2.0:1773941588.739267:0:14794:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:3.0:1773941588.739268:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941588.739274:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c2300. 00000001:00000001:2.0:1773941588.739275:0:14794:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:2.0:1773941588.739283:0:14794:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:3.0:1773941588.739284:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000001:00000001:2.0:1773941588.739286:0:14794:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941588.739289:0:14794:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941588.739291:0:14794:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941588.739293:0:14794:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941588.739300:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1773941588.739301:0:14794:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:3.0:1773941588.739310:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941588.739311:0:14794:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941588.739319:0:14794:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941588.739323:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941588.739324:0:14794:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941588.739329:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1773941588.739331:0:14794:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:3.0:1773941588.739336:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941588.739338:0:14794:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:3.0:1773941588.739341:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:01000000:2.0:1773941588.739343:0:14794:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:2.0:1773941588.739349:0:14794:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:2.0:1773941588.739375:0:14794:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:2.0:1773941588.739377:0:14794:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941588.739379:0:14794:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:2.0:1773941588.739384:0:14794:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:2.0:1773941588.739394:0:14794:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941588.739397:0:14794:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941588.739399:0:14794:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941588.739402:0:14794:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941588.739404:0:14794:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941588.742010:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d04dd4e00. 00010000:00000010:1.1:1773941588.742022:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d01488800. 00000020:00000001:2.0:1773941588.987205:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941588.987215:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941588.987221:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941590.011048:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294980331, at 4294980352 00000100:00000040:0.0:1773941590.011050:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294980331, at 4294980352 00000020:00000001:3.0:1773941590.011063:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000100:00000010:0.0:1773941590.011064:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3d20. 00000100:00000010:1.0:1773941590.011066:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3b40. 00000020:00000001:3.0:1773941590.011070:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941590.011076:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941590.331084:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941590.331091:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941590.331096:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941590.331100:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941590.331102:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941590.331103:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941590.331106:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941590.331127:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941590.331130:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:1.0:1773941590.331133:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941590.331135:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941590.331143:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c0700. 00000020:00000040:1.0:1773941590.331149:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941590.331155:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941590.331158:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941590.331163:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941590.331169:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d181fd800. 00000100:00000001:1.0:1773941590.331174:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941590.331178:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941590.331181:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941590.331188:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d04dd4e00. 00010000:00000001:1.0:1773941590.331192:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941590.331195:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d01488800. 00000020:00000001:1.0:1773941590.331199:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941590.331201:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d01488800 with handle 0x8e86767622e2a78 to hash 00000020:00000001:1.0:1773941590.331203:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941590.331204:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941590.331206:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941590.331209:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941590.331212:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941590.331215:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a78 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941590.331223:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a78 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941590.331232:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a78 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941590.331237:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941590.331238:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941590.331248:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941590.331249:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330528610752 : -113743180940864 : ffff988d1a1ab9c0) 00000100:00000001:1.0:1773941590.331252:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941590.331253:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00000100:00000001:1.0:1773941590.331257:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941590.331260:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941590.331262:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941590.331263:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941590.331272:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c0700 x1860112291494912/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941590.331279:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941590.331280:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941590.331283:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941590.331285:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941590.331287:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941590.331289:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291494912:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941590.331292:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941590.331296:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941590.331298:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941590.331300:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941590.331303:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d11188400. 02000000:00000001:1.0:1773941590.331305:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941590.331311:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941590.331314:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c5d8. 00000400:00000010:1.0:1773941590.331318:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000100:00000200:1.0:1773941590.331323:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291494912, portal 25 00000100:00000001:1.0:1773941590.331325:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941590.331326:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00000100:00000040:1.0:1773941590.331329:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c0700 x1860112291494912/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941606 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941590.331337:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941590.331340:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941590.331344:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de6e8. 00000100:00000200:1.0:1773941590.331352:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291494912, offset 0 00000400:00000200:1.0:1773941590.331359:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941590.331368:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941590.331376:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941590.331382:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941590.331386:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941590.331390:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941590.331393:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941590.331397:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[144, 143] 00000400:00000200:1.0:1773941590.331399:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:1.0:1773941590.331402:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [255:255:256:4294967295] 192.168.204.124@tcp LPNI seq info [145:145:8:4294967295] 00000400:00000200:1.0:1773941590.331406:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941590.331415:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:1.0:1773941590.331419:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:1.0:1773941590.331423:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfd300. 00000800:00000200:1.0:1773941590.331429:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:1.0:1773941590.331434:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:1.0:1773941590.331437:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfd300 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941590.331453:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941590.331457:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941590.331459:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294981927 00000100:00000010:1.0:1773941590.331462:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3b40. 00000100:00000001:1.0:1773941590.331467:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1773941590.331472:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294981927 00000100:00000010:1.0:1773941590.331474:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f36c0. 00000100:00000001:1.0:1773941590.331476:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1773941590.331479:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941590.331496:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941590.331498:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941590.331499:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941590.331510:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 11 seconds 00000100:00000001:1.0:1773941590.331513:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941590.331515:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941590.331611:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941590.331616:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfd300. 00000400:00000200:0.0:1773941590.331621:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941590.331626:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941590.331630:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de6e8 00000400:00000010:0.0:1773941590.331631:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de6e8. 00000100:00000001:0.0:1773941590.331634:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941590.331639:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c0700 x1860112291494912/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941606 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941590.331647:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941590.331650:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d203c0700 x1860112291494912/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941606 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941590.331655:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941590.331656:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941590.332837:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941590.332850:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941590.332852:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941590.332854:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941590.332858:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941590.332862:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941590.332866:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941590.332869:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941590.332873:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941590.332876:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324607c00 00000400:00000200:0.0:1773941590.332879:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941590.332882:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0x10d9 [1] + 224 00000800:00000001:0.0:1773941590.332887:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941590.332894:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941590.332896:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941590.332904:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941590.332908:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941590.332912:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c0700 x1860112291494912/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941606 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941590.332920:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d203c0700 x1860112291494912/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941606 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941590.332937:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941590.332944:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941590.332947:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941590.332986:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941590.332991:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941590.332992:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941590.332994:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:1.0:1773941590.332997:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:1.0:1773941590.332999:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:1.0:1773941590.333000:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941590.333005:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c0700 x1860112291494912/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941606 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941590.333012:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c0700 x1860112291494912/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941606 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941590.333018:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941590.333019:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941590.333021:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941590.333022:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941590.333025:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941590.333026:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941590.333028:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941590.333032:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=91 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941590.333035:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941590.333037:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941590.333040:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:1.0:1773941590.333043:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941590.333045:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941590.333048:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941590.333051:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941590.333056:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941590.333060:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941590.333063:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941590.333073:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941590.333076:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941590.333080:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d203c0700 x1860112291494912/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941606 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941590.333089:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941590.333090:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941590.333092:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941590.333095:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941590.333098:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941590.333100:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941590.333102:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941590.333103:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941590.333105:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941590.333106:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941590.333108:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941590.333111:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941590.333113:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941590.333116:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941590.333120:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 311 00000400:00000200:1.0:1773941590.333125:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941590.333129:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941590.333132:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941590.333135:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941590.333137:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941590.333139:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941590.333141:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941590.333143:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941590.333144:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941590.333146:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941590.333148:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941590.333150:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941590.333151:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941590.333153:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941590.333154:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941590.333160:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941590.333163:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941590.333165:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941590.333167:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941590.333169:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941590.333172:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941590.333174:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941590.333177:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941590.333179:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941590.333183:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c0380. 00000020:00000040:1.0:1773941590.333186:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941590.333190:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941590.333191:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941590.333194:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941590.333195:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941590.333198:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941590.333202:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d1118a400. 00000100:00000001:1.0:1773941590.333207:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941590.333214:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c0380 x1860112291495040/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941590.333223:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941590.333226:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941590.333231:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000da1038d4] to pc [ptlrpcd_rcv+-1] req@ffff988d203c0380 x1860112291495040/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941590.333247:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941590.333249:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941590.333251:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941590.333253:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941590.333255:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941590.333256:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941590.333257:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941590.333259:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941590.333260:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941590.333268:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c0700 x1860112291494912/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941606 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941590.333274:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941590.333277:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941590.333279:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941590.333280:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941590.333283:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c0700 x1860112291494912/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941606 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941590.333288:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291494912:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941590.333291:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941590.333293:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941590.333294:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941590.333295:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941590.333297:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941590.333299:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c0700 x1860112291494912/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941606 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941590.333305:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941590.333305:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941590.333310:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000010:1.0:1773941590.333310:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941590.333315:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941590.333317:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:1.0:1773941590.333320:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000040:3.0:1773941590.333322:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c0380 x1860112291495040/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941590.333323:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941590.333326:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941590.333328:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488800 refcount=4 00000020:00000001:1.0:1773941590.333331:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941590.333333:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00010000:1.0:1773941590.333336:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a78 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941590.333353:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941590.333359:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00010000:1.0:1773941590.333360:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a78 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941590.333363:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941590.333368:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941590.333373:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941590.333373:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941590.333377:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a78 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941590.333379:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941590.333385:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00100000:3.0:1773941590.333389:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291495040:192.168.204.114@tcp:250:ll_cfg_requeue.0 00010000:00000001:1.0:1773941590.333389:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941590.333393:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a78 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941590.333400:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00000001:1.0:1773941590.333401:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941590.333405:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941590.333406:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00000001:1.0:1773941590.333408:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 02000000:00000001:3.0:1773941590.333409:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941590.333411:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a78 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941590.333414:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00010000:1.0:1773941590.333420:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d01488800 already unused, calling callback (ffffffffc12b4c20) 10000000:00000001:1.0:1773941590.333422:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 02000000:00000010:3.0:1773941590.333423:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03520c00. 10000000:00010000:1.0:1773941590.333424:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a78 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941590.333429:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941590.333432:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941590.333434:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941590.333436:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:3.0:1773941590.333438:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000020:00000040:1.0:1773941590.333438:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488800 refcount=5 00000020:00000001:1.0:1773941590.333442:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941590.333445:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00000400:00000010:3.0:1773941590.333446:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00010000:00000001:1.0:1773941590.333447:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941590.333450:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a78 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:3.0:1773941590.333455:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 10000000:00000001:1.0:1773941590.333456:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000200:3.0:1773941590.333461:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291495040, portal 25 10000000:00010000:1.0:1773941590.333462:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a78 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941590.333466:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 10000000:01000000:1.0:1773941590.333469:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000100:00000001:3.0:1773941590.333471:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631455616 : -113743078096000 : ffff988d203c0380) 10000000:00000040:1.0:1773941590.333472:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941590.333475:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941590.333479:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941590.333494:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000100:00000040:3.0:1773941590.333498:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c0380 x1860112291495040/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941600 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941590.333499:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941590.333502:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941590.333505:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941590.333508:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941590.333514:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d01488800 with handle 0x8e86767622e2a78 from hash 00010000:00000001:1.0:1773941590.333518:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941590.333519:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941590.333522:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941590.333525:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941590.333528:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941590.333531:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000100:00000001:3.0:1773941590.333535:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:1.0:1773941590.333536:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941590.333540:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000040:3.0:1773941590.333543:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:3.0:1773941590.333548:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acb28. 00010000:00000001:1.0:1773941590.333552:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1773941590.333555:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291495040, offset 0 10000000:00000001:1.0:1773941590.333557:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941590.333564:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00010000:1.0:1773941590.333567:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a78 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941590.333572:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941590.333580:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:1.0:1773941590.333581:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941590.333585:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:3.0:1773941590.333586:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:1.0:1773941590.333588:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941590.333591:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000400:00000200:3.0:1773941590.333593:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00000001:1.0:1773941590.333594:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941590.333597:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941590.333599:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:3.0:1773941590.333602:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00010000:1.0:1773941590.333602:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a78 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941590.333610:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000040:1.0:1773941590.333614:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d04dd4e00 count: 0 00000400:00000200:3.0:1773941590.333616:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[144, 145] 00000400:00000200:3.0:1773941590.333623:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941590.333630:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [256:256:256:4294967295] 192.168.204.114@tcp LPNI seq info [146:146:8:4294967295] 00010000:00000001:1.0:1773941590.333634:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:3.0:1773941590.333636:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941590.333639:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941590.333644:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000400:00000200:3.0:1773941590.333654:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000100:00000040:1.0:1773941590.333654:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c0700 x1860112291494912/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941606 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1773941590.333668:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000100:00000001:1.0:1773941590.333669:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941590.333673:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000800:00000010:3.0:1773941590.333677:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74800. 02000000:00000010:1.0:1773941590.333677:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d11188400. 02000000:00000001:1.0:1773941590.333682:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941590.333685:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941590.333687:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941590.333689:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941590.333691:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d181fd800. 00000800:00000200:3.0:1773941590.333695:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 02000000:00000001:1.0:1773941590.333696:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941590.333699:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941590.333701:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c0700. 00000100:00000001:1.0:1773941590.333704:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941590.333706:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:3.0:1773941590.333708:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 10000000:00000001:1.0:1773941590.333709:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941590.333712:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941590.333715:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000800:00000200:3.0:1773941590.333716:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74800 type 1, nob 616 niov 1 nkiov 1 10000000:00000010:1.0:1773941590.333719:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d04d67a80. 00000020:00000010:1.0:1773941590.333724:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d181fd800. 00000020:00000010:1.0:1773941590.333727:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d181fc200. 00000040:00000010:1.0:1773941590.333730:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d181fc800. 00000100:00000001:3.0:1773941590.333732:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941590.333734:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941590.333740:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d181fce00. 00000400:00000200:3.0:1773941590.333741:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 80000000:00000010:1.0:1773941590.333744:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d054f35a0. 00000100:00000001:3.0:1773941590.333746:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294981929 00040000:00000010:1.0:1773941590.333750:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d181fd000. 20000000:00000010:1.0:1773941590.333755:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d181fc400. 00000100:00000001:3.0:1773941590.333756:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294981927 10000000:00000040:1.0:1773941590.333759:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:1.0:1773941590.333763:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941590.333764:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294981929 00000020:00000010:1.0:1773941590.333767:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d181fc200. 00000040:00000010:1.0:1773941590.333772:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d181fc800. 00000100:00000001:3.0:1773941590.333773:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294981927 00000100:00000001:3.0:1773941590.333780:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:1.0:1773941590.333781:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d18b18000. 00000100:00000001:3.0:1773941590.333787:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941590.333788:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d181fce00. 80000000:00000010:1.0:1773941590.333792:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d054f35a0. 00040000:00000010:1.0:1773941590.333796:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d181fd000. 20000000:00000010:1.0:1773941590.333801:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d181fc400. 00000100:00000001:3.0:1773941590.333802:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941590.333804:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d181fd800. 00000100:00000001:3.0:1773941590.333807:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:00000010:1.0:1773941590.333808:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d04d67a80. 00000800:00000200:0.0:1773941590.333810:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 10000000:01000000:1.0:1773941590.333815:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 00000800:00000010:0.0:1773941590.333815:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74800. 00000100:00000001:3.0:1773941590.333817:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 10000000:00000001:1.0:1773941590.333821:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941590.333822:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000400:00000200:0.0:1773941590.333823:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 10000000:00020000:1.0:1773941590.333826:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941590.333829:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 00000100:00000001:3.0:1773941590.333830:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000200:0.0:1773941590.333832:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 10000000:00000001:1.0:1773941590.333833:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 00000100:00000001:3.0:1773941590.333834:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 10000000:00000040:1.0:1773941590.333838:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 00000100:00000001:3.0:1773941590.333839:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941590.333839:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acb28 00000400:00000010:0.0:1773941590.333843:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acb28. 10000000:00000001:1.0:1773941590.333845:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000100:00000001:3.0:1773941590.333846:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941590.333847:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941590.333854:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c0380 x1860112291495040/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941600 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941590.333860:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941590.333863:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c0380 x1860112291495040/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941600 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941590.333868:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941590.333870:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941590.335195:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941590.335208:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941590.335211:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941590.335214:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941590.335219:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941590.335225:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941590.335231:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941590.335234:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941590.335238:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941590.335249:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324607c80 00000400:00000200:0.0:1773941590.335254:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941590.335259:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0x10e9 [1] + 0 00000800:00000001:0.0:1773941590.335265:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941590.335272:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941590.335275:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941590.335279:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941590.335284:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941590.335289:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c0380 x1860112291495040/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941600 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941590.335301:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c0380 x1860112291495040/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941600 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941590.335319:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941590.335326:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941590.335330:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941590.335338:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941590.335342:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941590.335347:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941590.335349:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941590.335352:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941590.335357:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941590.335358:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941590.335360:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941590.335364:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c0380 x1860112291495040/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941600 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941590.335371:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c0380 x1860112291495040/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941600 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941590.335375:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941590.335376:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941590.335378:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941590.335379:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941590.335380:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941590.335382:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941590.335383:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941590.335387:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=91 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941590.335389:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941590.335391:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941590.335392:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941590.335393:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941590.335395:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941590.335397:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941590.335399:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c0380 x1860112291495040/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941600 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941590.335413:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941590.335415:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941590.335416:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941590.335419:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941590.335423:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941590.335425:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941590.335426:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941590.335428:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941590.335430:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941590.335431:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941590.335433:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941590.335436:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941590 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941590.335438:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941590.335443:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941590.335445:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941590.335446:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941590.335448:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941590.335449:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941590.335452:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941590.335454:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941590.335456:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941590.335459:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941590.335461:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941590.335463:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941590.335724:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941590.335728:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941590.335735:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c0380 x1860112291495040/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941600 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941590.335744:0:14795:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:2.0:1773941590.335745:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291495040:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:2.0:1773941590.335750:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1773941590.335752:0:14795:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00000001:2.0:1773941590.335752:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00100000:3.0:1773941590.335755:0:14795:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:3.0:1773941590.335757:0:14795:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941590.335759:0:14795:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000040:2.0:1773941590.335761:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c0380 x1860112291495040/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941600 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000001:00000001:3.0:1773941590.335762:0:14795:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941590.335765:0:14795:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941590.335768:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941590.335770:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:3.0:1773941590.335773:0:14795:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941590.335773:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03520c00. 00000001:00000001:3.0:1773941590.335775:0:14795:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:3.0:1773941590.335776:0:14795:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941590.335776:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:3.0:1773941590.335778:0:14795:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941590.335778:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941590.335779:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 10000000:00000001:3.0:1773941590.335780:0:14795:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941590.335782:0:14795:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000001:2.0:1773941590.335782:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941590.335784:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d1118a400. 00000100:00080000:3.0:1773941590.335785:0:14795:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 02000000:00000001:2.0:1773941590.335786:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1773941590.335788:0:14795:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 02000000:00000001:2.0:1773941590.335788:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1773941590.335790:0:14795:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000010:2.0:1773941590.335790:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c0380. 00000100:00000001:3.0:1773941590.335793:0:14795:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941590.335793:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941590.335794:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1773941590.335796:0:14795:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941590.335797:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941590.335798:0:14795:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941590.335799:0:14795:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000100:00000001:2.0:1773941590.335800:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 10000000:00000040:3.0:1773941590.335801:0:14795:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00000001:2.0:1773941590.335802:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941590.335804:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941590.335805:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1773941590.335816:0:14795:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:3.0:1773941590.335818:0:14795:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941590.335820:0:14795:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:3.0:1773941590.335823:0:14795:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:3.0:1773941590.335827:0:14795:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941590.335829:0:14795:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941590.335832:0:14795:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941590.335834:0:14795:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941590.335836:0:14795:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941590.338992:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d04dd4e00. 00010000:00000010:1.1:1773941590.339001:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d01488800. 00000020:00000001:2.0:1773941591.035143:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941591.035149:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941591.035152:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1773941591.611041:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294981929, at 4294981952 00000100:00000040:1.0:1773941591.611043:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294981929, at 4294981952 00000100:00000010:1.0:1773941591.611059:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f36c0. 00000100:00000010:0.0:1773941591.611071:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3b40. 00000020:00000001:2.0:1773941592.059093:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941592.059098:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941592.059102:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941592.251041:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941592.251061:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941592.251065:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941592.251068:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941592.251070:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941592.251071:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941592.251073:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941592.251076:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941592.251078:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:1.0:1773941592.251081:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941592.251083:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941592.251094:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c0700. 00000020:00000040:1.0:1773941592.251099:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941592.251105:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941592.251109:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941592.251126:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941592.251134:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d181fd000. 00000100:00000001:1.0:1773941592.251140:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941592.251144:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941592.251161:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941592.251167:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d04dd4e00. 00010000:00000001:1.0:1773941592.251172:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941592.251186:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d01488800. 00000020:00000001:1.0:1773941592.251190:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941592.251192:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d01488800 with handle 0x8e86767622e2a7f to hash 00000020:00000001:1.0:1773941592.251194:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941592.251195:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941592.251198:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941592.251201:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941592.251204:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941592.251207:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a7f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941592.251215:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a7f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941592.251225:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a7f lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941592.251229:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941592.251231:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941592.251236:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941592.251237:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330528610752 : -113743180940864 : ffff988d1a1ab9c0) 00000100:00000001:1.0:1773941592.251240:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941592.251241:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00000100:00000001:1.0:1773941592.251245:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941592.251248:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941592.251250:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941592.251251:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941592.251260:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c0700 x1860112291495168/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941592.251267:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941592.251272:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941592.251287:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941592.251289:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941592.251291:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941592.251294:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291495168:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941592.251297:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941592.251301:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941592.251303:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941592.251305:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941592.251308:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d11188400. 02000000:00000001:1.0:1773941592.251310:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941592.251316:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941592.251319:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c5d8. 00000400:00000010:1.0:1773941592.251324:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000100:00000200:1.0:1773941592.251329:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291495168, portal 25 00000100:00000001:1.0:1773941592.251331:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941592.251332:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00000100:00000040:1.0:1773941592.251335:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c0700 x1860112291495168/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941608 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941592.251352:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941592.251354:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941592.251355:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113deee0. 00000100:00000200:1.0:1773941592.251357:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291495168, offset 0 00000400:00000200:1.0:1773941592.251362:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941592.251368:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941592.251374:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941592.251378:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941592.251382:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941592.251385:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941592.251387:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941592.251390:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[146, 145] 00000400:00000200:1.0:1773941592.251392:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:1.0:1773941592.251395:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [257:257:256:4294967295] 192.168.204.124@tcp LPNI seq info [147:147:8:4294967295] 00000400:00000200:1.0:1773941592.251402:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941592.251410:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:1.0:1773941592.251415:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:1.0:1773941592.251419:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfdd00. 00000800:00000200:1.0:1773941592.251425:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:1.0:1773941592.251431:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:1.0:1773941592.251435:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfdd00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941592.251472:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941592.251476:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941592.251479:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294983847 00000100:00000010:1.0:1773941592.251482:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f36c0. 00000100:00000001:1.0:1773941592.251488:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1773941592.251495:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294983847 00000100:00000010:1.0:1773941592.251509:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f35a0. 00000100:00000001:1.0:1773941592.251511:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1773941592.251514:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941592.251516:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941592.251518:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941592.251519:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941592.251521:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 11 seconds 00000100:00000001:1.0:1773941592.251523:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941592.251525:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941592.251602:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941592.251607:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfdd00. 00000400:00000200:0.0:1773941592.251611:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941592.251616:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941592.251620:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113deee0 00000400:00000010:0.0:1773941592.251622:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113deee0. 00000100:00000001:0.0:1773941592.251625:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941592.251629:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c0700 x1860112291495168/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941608 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941592.251638:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941592.251641:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d203c0700 x1860112291495168/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941608 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941592.251646:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941592.251647:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941592.253263:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941592.253286:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941592.253288:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941592.253290:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941592.253294:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941592.253297:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941592.253302:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941592.253305:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941592.253308:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941592.253311:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324607d00 00000400:00000200:0.0:1773941592.253314:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941592.253317:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0x10f1 [1] + 224 00000800:00000001:0.0:1773941592.253322:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941592.253328:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941592.253330:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941592.253332:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941592.253347:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941592.253350:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c0700 x1860112291495168/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941608 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941592.253358:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d203c0700 x1860112291495168/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941608 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941592.253371:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941592.253377:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941592.253379:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941592.253459:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941592.253465:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941592.253466:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941592.253468:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:1.0:1773941592.253472:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:1.0:1773941592.253473:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:1.0:1773941592.253475:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941592.253480:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c0700 x1860112291495168/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941608 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941592.253488:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c0700 x1860112291495168/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941608 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941592.253499:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941592.253501:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941592.253503:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941592.253505:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941592.253507:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941592.253509:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941592.253511:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941592.253516:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=93 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941592.253518:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941592.253521:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941592.253524:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:1.0:1773941592.253526:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941592.253528:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941592.253531:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941592.253534:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941592.253537:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941592.253540:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941592.253542:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941592.253543:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941592.253544:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941592.253547:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d203c0700 x1860112291495168/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941608 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941592.253552:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941592.253553:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941592.253554:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941592.253556:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941592.253558:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941592.253560:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941592.253561:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941592.253562:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941592.253563:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941592.253564:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941592.253567:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941592.253569:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941592.253571:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941592.253573:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941592.253576:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 313 00000400:00000200:1.0:1773941592.253579:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941592.253582:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941592.253585:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941592.253588:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941592.253590:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941592.253591:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941592.253594:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941592.253595:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941592.253597:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941592.253599:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941592.253600:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941592.253602:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941592.253603:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941592.253604:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941592.253605:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941592.253607:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941592.253609:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941592.253611:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941592.253613:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941592.253615:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941592.253617:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941592.253619:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941592.253621:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941592.253622:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941592.253624:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c2a00. 00000020:00000040:1.0:1773941592.253626:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941592.253628:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941592.253632:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941592.253635:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941592.253636:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941592.253638:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941592.253641:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d1118a800. 00000100:00000001:1.0:1773941592.253643:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941592.253648:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c2a00 x1860112291495296/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941592.253653:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941592.253656:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941592.253659:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000fb7523b4] to pc [ptlrpcd_rcv+-1] req@ffff988d203c2a00 x1860112291495296/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941592.253677:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941592.253679:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941592.253680:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941592.253682:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941592.253684:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941592.253685:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941592.253686:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941592.253688:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941592.253689:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941592.253692:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c0700 x1860112291495168/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941608 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941592.253697:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941592.253699:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941592.253701:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941592.253702:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941592.253706:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c0700 x1860112291495168/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941608 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941592.253712:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291495168:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941592.253715:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941592.253722:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941592.253724:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941592.253724:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941592.253728:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941592.253730:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1773941592.253731:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c0700 x1860112291495168/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941608 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941592.253735:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941592.253740:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941592.253743:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941592.253746:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000040:2.0:1773941592.253747:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c2a00 x1860112291495296/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941592.253748:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:1.0:1773941592.253752:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941592.253756:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941592.253759:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941592.253762:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488800 refcount=4 00000100:00000001:2.0:1773941592.253766:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000001:1.0:1773941592.253766:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00000100:00000001:2.0:1773941592.253777:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941592.253777:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 02000000:00000001:2.0:1773941592.253784:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00010000:00010000:1.0:1773941592.253784:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a7f lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941592.253786:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941592.253792:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941592.253794:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00010000:00010000:1.0:1773941592.253798:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a7f lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941592.253807:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00100000:2.0:1773941592.253808:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291495296:192.168.204.114@tcp:250:ll_cfg_requeue.0 00010000:00010000:1.0:1773941592.253810:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a7f lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941592.253819:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00000001:1.0:1773941592.253821:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 02000000:00000001:2.0:1773941592.253823:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00000001:1.0:1773941592.253823:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 02000000:00000001:2.0:1773941592.253827:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941592.253828:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a7f lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941592.253833:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941592.253844:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a7c00. 00010000:00000001:1.0:1773941592.253844:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 02000000:00000001:2.0:1773941592.253848:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941592.253853:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941592.253857:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941592.253859:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941592.253862:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a7f lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941592.253863:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000400:00000010:2.0:1773941592.253871:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00010000:00010000:1.0:1773941592.253871:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d01488800 already unused, calling callback (ffffffffc12b4c20) 10000000:00000001:1.0:1773941592.253876:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000200:2.0:1773941592.253878:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291495296, portal 25 10000000:00010000:1.0:1773941592.253879:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a7f lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941592.253884:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00000001:1.0:1773941592.253887:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000100:00000001:2.0:1773941592.253888:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631465472 : -113743078086144 : ffff988d203c2a00) 00010000:00000001:1.0:1773941592.253889:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941592.253892:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941592.253894:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488800 refcount=5 00000020:00000001:1.0:1773941592.253897:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00000100:00000040:2.0:1773941592.253898:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c2a00 x1860112291495296/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941602 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941592.253911:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941592.253917:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941592.253920:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a7f lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941592.253925:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 10000000:00000001:1.0:1773941592.253930:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000040:2.0:1773941592.253931:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 10000000:00010000:1.0:1773941592.253933:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a7f lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941592.253936:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516770. 10000000:01000000:1.0:1773941592.253938:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000100:00000200:2.0:1773941592.253942:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291495296, offset 0 10000000:00000040:1.0:1773941592.253942:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941592.253944:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941592.253948:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941592.253950:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:2.0:1773941592.253951:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00000001:1.0:1773941592.253953:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941592.253956:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000400:00000200:2.0:1773941592.253959:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941592.253977:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941592.253979:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941592.253981:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d01488800 with handle 0x8e86767622e2a7f from hash 00010000:00000001:1.0:1773941592.253987:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941592.253988:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941592.253989:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941592.253991:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941592.253992:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941592.253994:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941592.253996:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941592.253997:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941592.253998:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941592.254000:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941592.254003:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a7f lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941592.254007:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941592.254009:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941592.254012:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941592.254014:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941592.254017:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:2.0:1773941592.254018:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:1.0:1773941592.254019:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941592.254022:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:2.0:1773941592.254025:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00010000:1.0:1773941592.254025:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a7f lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941592.254032:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00000040:1.0:1773941592.254035:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d04dd4e00 count: 0 00000400:00000200:2.0:1773941592.254037:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941592.254052:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941592.254058:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[146, 147] 00000400:00000200:2.0:1773941592.254063:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00010000:00000001:1.0:1773941592.254064:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941592.254069:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000200:2.0:1773941592.254070:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [258:258:256:4294967295] 192.168.204.114@tcp LPNI seq info [148:148:8:4294967295] 00000100:00000001:1.0:1773941592.254075:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000400:00000200:2.0:1773941592.254079:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000040:1.0:1773941592.254084:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c0700 x1860112291495168/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941608 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000200:2.0:1773941592.254094:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000100:00000001:1.0:1773941592.254102:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941592.254106:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941592.254110:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d11188400. 00000800:00000200:2.0:1773941592.254116:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 02000000:00000001:1.0:1773941592.254126:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000800:00000010:2.0:1773941592.254128:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0db000. 00000020:00000001:1.0:1773941592.254130:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941592.254133:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000800:00000200:2.0:1773941592.254138:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000020:00000001:1.0:1773941592.254139:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 00000800:00000200:2.0:1773941592.254148:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 02000000:00000010:1.0:1773941592.254148:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d181fd000. 02000000:00000001:1.0:1773941592.254153:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000800:00000200:2.0:1773941592.254155:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0db000 type 1, nob 616 niov 1 nkiov 1 02000000:00000001:1.0:1773941592.254156:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941592.254159:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c0700. 00000100:00000001:1.0:1773941592.254162:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941592.254163:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941592.254164:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941592.254166:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941592.254168:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000100:00000001:2.0:1773941592.254169:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 10000000:00000010:1.0:1773941592.254171:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d04d67580. 00000020:00000010:1.0:1773941592.254175:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d181fd000. 00000400:00000200:2.0:1773941592.254176:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941592.254180:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294983850 00000020:00000010:1.0:1773941592.254192:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d181fce00. 00000100:00000001:2.0:1773941592.254196:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294983847 00000040:00000010:1.0:1773941592.254197:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d181fc800. 00000100:00000001:2.0:1773941592.254203:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294983850 00000020:00000010:1.0:1773941592.254203:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941592.254210:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d181fc200. 00000100:00000001:2.0:1773941592.254211:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294983847 80000000:00000010:1.0:1773941592.254216:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d054f33c0. 00000100:00000001:2.0:1773941592.254217:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941592.254222:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00040000:00000010:1.0:1773941592.254224:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d181fc400. 00000800:00000200:0.0:1773941592.254224:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1773941592.254225:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1773941592.254229:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0db000. 00000100:00000001:2.0:1773941592.254232:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 20000000:00000010:1.0:1773941592.254232:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d181fd800. 00000400:00000200:0.0:1773941592.254234:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 10000000:00000040:1.0:1773941592.254240:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 00000100:00000001:2.0:1773941592.254243:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000400:00000200:0.0:1773941592.254243:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 10000000:00000001:1.0:1773941592.254244:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941592.254248:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000400:00000200:0.0:1773941592.254250:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516770 00000020:00000010:1.0:1773941592.254251:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d181fce00. 00000100:00000001:2.0:1773941592.254252:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000400:00000010:0.0:1773941592.254253:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516770. 00000100:00000001:2.0:1773941592.254256:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000040:00000010:1.0:1773941592.254257:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d181fc800. 00000100:00000001:0.0:1773941592.254258:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1773941592.254261:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941592.254263:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d18b18000. 00000100:00000200:0.0:1773941592.254266:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c2a00 x1860112291495296/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941602 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941592.254276:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941592.254281:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d181fc200. 80000000:00000010:1.0:1773941592.254287:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d054f33c0. 00040000:00000010:1.0:1773941592.254291:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d181fc400. 20000000:00000010:1.0:1773941592.254295:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d181fd800. 00000020:00000010:1.0:1773941592.254300:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d181fd000. 10000000:00000010:1.0:1773941592.254304:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d04d67580. 00000100:00000001:0.0:1773941592.254305:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 10000000:01000000:1.0:1773941592.254307:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 00000100:00000040:0.0:1773941592.254311:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c2a00 x1860112291495296/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941602 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000001:1.0:1773941592.254312:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941592.254317:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941592.254320:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941592.254323:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 00000100:00000001:0.0:1773941592.254324:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941592.254326:0:2393:0:(events.c:68:request_out_callback()) Process leaving 10000000:00000040:1.0:1773941592.254330:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941592.254343:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941592.256660:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941592.256668:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941592.256670:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941592.256672:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941592.256675:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941592.256678:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941592.256683:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941592.256685:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941592.256688:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941592.256690:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324607d80 00000400:00000200:0.0:1773941592.256693:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941592.256695:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0x1101 [1] + 0 00000800:00000001:0.0:1773941592.256699:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941592.256711:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941592.256713:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941592.256715:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941592.256718:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941592.256721:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c2a00 x1860112291495296/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941602 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941592.256729:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c2a00 x1860112291495296/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941602 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941592.256743:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941592.256747:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941592.256749:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941592.256778:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941592.256781:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941592.256784:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941592.256785:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941592.256788:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941592.256792:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941592.256794:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941592.256796:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941592.256802:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c2a00 x1860112291495296/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941602 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941592.256812:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c2a00 x1860112291495296/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941602 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941592.256839:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941592.256841:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941592.256843:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941592.256845:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941592.256847:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941592.256849:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941592.256851:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941592.256855:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=93 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941592.256858:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941592.256861:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941592.256863:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941592.256865:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941592.256867:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941592.256870:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941592.256874:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c2a00 x1860112291495296/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941602 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941592.256888:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941592.256890:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941592.256891:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941592.256894:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941592.256898:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941592.256900:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941592.256902:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941592.256904:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941592.256906:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941592.256909:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941592.256911:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:2.0:1773941592.256915:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941592 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941592.256918:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941592.256923:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:2.0:1773941592.256925:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941592.256927:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941592.256929:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941592.256936:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941592.256940:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:2.0:1773941592.256942:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941592.256945:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941592.256948:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941592.256952:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:2.0:1773941592.256955:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941592.257198:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941592.257202:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941592.257208:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c2a00 x1860112291495296/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941602 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941592.257216:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c2a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291495296:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941592.257221:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941592.257223:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941592.257225:0:14797:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000040:3.0:1773941592.257226:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c2a00 x1860112291495296/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941602 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941592.257233:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00080000:2.0:1773941592.257236:0:14797:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:2.0:1773941592.257240:0:14797:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000001:3.0:1773941592.257242:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1773941592.257244:0:14797:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000010:3.0:1773941592.257246:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a7c00. 10000000:01000000:2.0:1773941592.257248:0:14797:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000001:3.0:1773941592.257250:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:2.0:1773941592.257253:0:14797:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:3.0:1773941592.257254:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941592.257257:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000001:00000001:2.0:1773941592.257258:0:14797:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000001:3.0:1773941592.257259:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941592.257263:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d1118a800. 00000001:00000001:2.0:1773941592.257264:0:14797:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941592.257267:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000001:00000001:2.0:1773941592.257268:0:14797:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:3.0:1773941592.257271:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000001:00000001:2.0:1773941592.257271:0:14797:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941592.257274:0:14797:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941592.257275:0:14797:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941592.257285:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c2a00. 00000100:00000001:2.0:1773941592.257285:0:14797:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941592.257291:0:14797:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:3.0:1773941592.257293:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941592.257295:0:14797:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:3.0:1773941592.257296:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941592.257299:0:14797:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941592.257301:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941592.257303:0:14797:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941592.257308:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1773941592.257308:0:14797:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941592.257311:0:14797:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:3.0:1773941592.257313:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 10000000:01000000:2.0:1773941592.257315:0:14797:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000100:00000001:3.0:1773941592.257318:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 10000000:00000040:2.0:1773941592.257318:0:14797:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00000001:3.0:1773941592.257322:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1773941592.257334:0:14797:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:2.0:1773941592.257338:0:14797:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941592.257348:0:14797:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:2.0:1773941592.257353:0:14797:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:2.0:1773941592.257356:0:14797:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941592.257359:0:14797:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941592.257366:0:14797:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941592.257368:0:14797:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941592.257370:0:14797:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941592.260008:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d04dd4e00. 00010000:00000010:1.1:1773941592.260016:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d01488800. 00000100:00000040:3.0:1773941592.315047:0:8:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (320) 00000020:00000001:2.0:1773941593.083128:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941593.083134:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941593.083138:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941593.531090:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294983850, at 4294983872 00000100:00000040:0.0:1773941593.531090:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294983850, at 4294983872 00000100:00000010:1.0:1773941593.531105:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f35a0. 00000100:00000010:0.0:1773941593.531123:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f36c0. 00000020:00000001:2.0:1773941594.107112:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941594.107118:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941594.107122:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941594.171058:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941594.171065:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941594.171068:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941594.171071:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941594.171072:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941594.171073:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941594.171076:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941594.171077:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941594.171079:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:1.0:1773941594.171082:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941594.171083:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941594.171092:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c0700. 00000020:00000040:1.0:1773941594.171095:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941594.171100:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941594.171102:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941594.171107:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941594.171113:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d181fd800. 00000100:00000001:1.0:1773941594.171119:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941594.171122:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941594.171125:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941594.171131:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d04dd4e00. 00010000:00000001:1.0:1773941594.171136:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941594.171138:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d01488800. 00000020:00000001:1.0:1773941594.171142:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941594.171144:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d01488800 with handle 0x8e86767622e2a86 to hash 00000020:00000001:1.0:1773941594.171145:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941594.171146:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941594.171149:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941594.171152:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941594.171154:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941594.171157:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a86 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941594.171180:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a86 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941594.171189:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a86 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941594.171194:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941594.171195:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941594.171200:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941594.171202:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330528610752 : -113743180940864 : ffff988d1a1ab9c0) 00000100:00000001:1.0:1773941594.171204:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941594.171205:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00000100:00000001:1.0:1773941594.171208:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941594.171212:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941594.171213:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941594.171215:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941594.171223:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c0700 x1860112291495424/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941594.171231:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941594.171232:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941594.171235:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941594.171237:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941594.171239:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941594.171241:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291495424:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941594.171244:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941594.171248:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941594.171250:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941594.171252:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941594.171255:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d11188400. 02000000:00000001:1.0:1773941594.171257:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941594.171262:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941594.171269:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c5d8. 00000400:00000010:1.0:1773941594.171274:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000100:00000200:1.0:1773941594.171280:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291495424, portal 25 00000100:00000001:1.0:1773941594.171282:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941594.171283:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00000100:00000040:1.0:1773941594.171287:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c0700 x1860112291495424/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941610 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941594.171292:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941594.171294:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941594.171296:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de088. 00000100:00000200:1.0:1773941594.171310:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291495424, offset 0 00000400:00000200:1.0:1773941594.171315:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941594.171322:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941594.171328:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941594.171332:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941594.171335:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941594.171339:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941594.171341:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941594.171344:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[148, 147] 00000400:00000200:1.0:1773941594.171346:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:1.0:1773941594.171349:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [259:259:256:4294967295] 192.168.204.124@tcp LPNI seq info [149:149:8:4294967295] 00000400:00000200:1.0:1773941594.171352:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941594.171360:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:1.0:1773941594.171364:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:1.0:1773941594.171368:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfd400. 00000800:00000200:1.0:1773941594.171374:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:1.0:1773941594.171379:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:1.0:1773941594.171383:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfd400 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941594.171399:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941594.171402:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941594.171405:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294985767 00000100:00000010:1.0:1773941594.171423:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f35a0. 00000100:00000001:1.0:1773941594.171429:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1773941594.171436:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294985767 00000100:00000010:1.0:1773941594.171438:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f33c0. 00000100:00000001:1.0:1773941594.171441:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1773941594.171443:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941594.171445:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941594.171447:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941594.171449:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941594.171451:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 11 seconds 00000100:00000001:1.0:1773941594.171454:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941594.171457:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941594.171558:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941594.171563:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfd400. 00000400:00000200:0.0:1773941594.171568:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941594.171573:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941594.171578:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de088 00000400:00000010:0.0:1773941594.171579:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de088. 00000100:00000001:0.0:1773941594.171583:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941594.171587:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c0700 x1860112291495424/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941610 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941594.171595:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941594.171597:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d203c0700 x1860112291495424/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941610 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941594.171602:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941594.171604:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941594.172726:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941594.172744:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941594.172747:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941594.172748:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941594.172752:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941594.172756:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941594.172761:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941594.172779:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941594.172782:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941594.172785:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324607e00 00000400:00000200:0.0:1773941594.172788:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941594.172792:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0x1109 [1] + 224 00000800:00000001:0.0:1773941594.172797:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941594.172804:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941594.172805:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941594.172808:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941594.172811:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941594.172820:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c0700 x1860112291495424/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941610 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941594.172827:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d203c0700 x1860112291495424/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941610 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941594.172841:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941594.172865:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941594.172869:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941594.172945:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941594.172953:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941594.172955:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941594.172957:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:1.0:1773941594.172983:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:1.0:1773941594.172986:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:1.0:1773941594.172989:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941594.172995:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c0700 x1860112291495424/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941610 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941594.173007:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c0700 x1860112291495424/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941610 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941594.173015:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941594.173016:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941594.173019:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941594.173021:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941594.173023:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941594.173025:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941594.173027:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941594.173031:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=95 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941594.173034:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941594.173036:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941594.173039:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:1.0:1773941594.173048:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941594.173051:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941594.173053:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941594.173057:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941594.173060:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941594.173063:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941594.173065:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941594.173066:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941594.173067:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941594.173070:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d203c0700 x1860112291495424/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941610 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941594.173075:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941594.173076:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941594.173077:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941594.173079:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941594.173081:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941594.173083:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941594.173084:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941594.173085:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941594.173085:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941594.173086:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941594.173087:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941594.173089:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941594.173090:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941594.173092:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941594.173094:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 315 00000400:00000200:1.0:1773941594.173097:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941594.173100:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941594.173103:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941594.173106:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941594.173107:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941594.173109:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941594.173114:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941594.173116:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941594.173117:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941594.173119:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941594.173121:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941594.173122:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941594.173124:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941594.173126:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941594.173127:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941594.173128:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941594.173130:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941594.173132:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941594.173134:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941594.173135:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941594.173138:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941594.173140:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941594.173141:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941594.173143:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941594.173145:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c3480. 00000020:00000040:1.0:1773941594.173147:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941594.173149:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941594.173150:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941594.173152:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941594.173153:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941594.173154:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941594.173157:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d11188c00. 00000100:00000001:1.0:1773941594.173160:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941594.173164:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c3480 x1860112291495552/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941594.173170:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941594.173172:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941594.173175:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000099ddc7e5] to pc [ptlrpcd_rcv+-1] req@ffff988d203c3480 x1860112291495552/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941594.173198:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941594.173200:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941594.173201:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941594.173203:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941594.173204:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941594.173206:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941594.173207:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941594.173208:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941594.173210:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941594.173213:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c0700 x1860112291495424/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941610 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941594.173218:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941594.173220:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941594.173222:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941594.173223:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941594.173226:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c0700 x1860112291495424/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941610 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941594.173233:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291495424:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941594.173237:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941594.173238:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941594.173240:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941594.173241:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941594.173243:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c0700 x1860112291495424/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941610 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941594.173247:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941594.173249:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941594.173251:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941594.173254:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941594.173255:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:1.0:1773941594.173256:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941594.173264:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000001:3.0:1773941594.173265:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1773941594.173266:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941594.173274:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488800 refcount=4 00000100:00000001:3.0:1773941594.173275:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000020:00000001:1.0:1773941594.173282:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941594.173285:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00010000:1.0:1773941594.173288:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a86 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:3.0:1773941594.173295:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c3480 x1860112291495552/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941594.173299:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00010000:00010000:1.0:1773941594.173308:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a86 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941594.173335:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:1.0:1773941594.173337:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000001:3.0:1773941594.173340:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941594.173344:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a86 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941594.173347:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941594.173352:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941594.173362:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941594.173363:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941594.173369:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00100000:3.0:1773941594.173371:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291495552:192.168.204.114@tcp:250:ll_cfg_requeue.0 00010000:00010000:1.0:1773941594.173375:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a86 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941594.173381:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941594.173389:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941594.173393:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941594.173396:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 02000000:00000001:3.0:1773941594.173399:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:1.0:1773941594.173401:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941594.173406:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 02000000:00000010:3.0:1773941594.173422:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03521c00. 00010000:00010000:1.0:1773941594.173427:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a86 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941594.173428:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941594.173441:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d01488800 already unused, calling callback (ffffffffc12b4c20) 10000000:00000001:1.0:1773941594.173446:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941594.173459:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a86 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941594.173463:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941594.173467:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941594.173477:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941594.173478:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941594.173480:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488800 refcount=5 00000020:00000001:1.0:1773941594.173487:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941594.173490:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00000400:00000010:3.0:1773941594.173493:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00010000:00000001:1.0:1773941594.173493:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941594.173498:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a86 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:3.0:1773941594.173505:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 10000000:00000001:1.0:1773941594.173506:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941594.173511:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a86 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1773941594.173516:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000100:00000200:3.0:1773941594.173518:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291495552, portal 25 10000000:00000040:1.0:1773941594.173519:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941594.173521:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941594.173529:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00010000:1.0:1773941594.173529:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000100:00000001:3.0:1773941594.173536:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631468160 : -113743078083456 : ffff988d203c3480) 00010000:00000001:1.0:1773941594.173536:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941594.173540:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941594.173549:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941594.173551:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941594.173554:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941594.173558:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c3480 x1860112291495552/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941604 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000040:1.0:1773941594.173559:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d01488800 with handle 0x8e86767622e2a86 from hash 00010000:00000001:1.0:1773941594.173565:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941594.173568:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941594.173573:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941594.173577:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941594.173581:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941594.173587:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941594.173608:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941594.173609:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941594.173610:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941594.173617:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941594.173619:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00010000:1.0:1773941594.173623:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a86 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:3.0:1773941594.173631:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:3.0:1773941594.173640:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acaa0. 00000100:00000200:3.0:1773941594.173653:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291495552, offset 0 00010000:00000001:1.0:1773941594.173654:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941594.173657:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941594.173660:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941594.173663:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941594.173664:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941594.173665:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941594.173666:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941594.173668:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a86 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941594.173674:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00000040:1.0:1773941594.173682:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d04dd4e00 count: 0 00000400:00000200:3.0:1773941594.173690:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941594.173692:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941594.173697:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941594.173701:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000400:00000200:3.0:1773941594.173705:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000100:00000040:1.0:1773941594.173710:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c0700 x1860112291495424/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941610 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941594.173730:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941594.173742:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000100:00000001:1.0:1773941594.173742:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941594.173748:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941594.173751:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d11188400. 00000400:00000200:3.0:1773941594.173756:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 02000000:00000001:1.0:1773941594.173758:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941594.173765:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000400:00000200:3.0:1773941594.173768:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000020:00000040:1.0:1773941594.173769:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941594.173779:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941594.173785:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d181fd800. 00000400:00000200:3.0:1773941594.173786:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[148, 149] 02000000:00000001:1.0:1773941594.173789:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941594.173792:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941594.173795:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c0700. 00000100:00000001:1.0:1773941594.173802:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000400:00000200:3.0:1773941594.173805:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000100:00000001:1.0:1773941594.173805:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941594.173815:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941594.173821:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000400:00000200:3.0:1773941594.173825:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [260:260:256:4294967295] 192.168.204.114@tcp LPNI seq info [150:150:8:4294967295] 10000000:00000001:1.0:1773941594.173826:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941594.173831:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d04d67c40. 00000020:00000010:1.0:1773941594.173838:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d181fd800. 00000400:00000200:3.0:1773941594.173839:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000020:00000010:1.0:1773941594.173862:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d181fc400. 00000040:00000010:1.0:1773941594.173868:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d181fc200. 00000400:00000200:3.0:1773941594.173879:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000020:00000010:1.0:1773941594.173879:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941594.173891:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d181fc800. 80000000:00000010:1.0:1773941594.173896:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d054f3420. 00000800:00000200:3.0:1773941594.173898:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00040000:00000010:1.0:1773941594.173902:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d181fce00. 20000000:00000010:1.0:1773941594.173908:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d181fd600. 00000800:00000010:3.0:1773941594.173909:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74800. 10000000:00000040:1.0:1773941594.173913:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:1.0:1773941594.173917:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000800:00000200:3.0:1773941594.173936:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000020:00000010:1.0:1773941594.173942:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d181fc400. 00000040:00000010:1.0:1773941594.173946:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d181fc200. 00000020:00000010:1.0:1773941594.173956:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d18b18000. 00000800:00000200:3.0:1773941594.173957:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000020:00000010:1.0:1773941594.173964:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d181fc800. 00000800:00000200:3.0:1773941594.173968:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74800 type 1, nob 616 niov 1 nkiov 1 80000000:00000010:1.0:1773941594.173969:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d054f3420. 00040000:00000010:1.0:1773941594.173976:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d181fce00. 20000000:00000010:1.0:1773941594.173980:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d181fd600. 00000020:00000010:1.0:1773941594.173983:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d181fd800. 10000000:00000010:1.0:1773941594.173985:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d04d67c40. 10000000:01000000:1.0:1773941594.173986:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941594.173988:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941594.173990:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941594.173991:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941594.173992:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941594.173993:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941594.173995:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000100:00000001:3.0:1773941594.174044:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941594.174054:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:3.0:1773941594.174058:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294985770 00000800:00000200:0.0:1773941594.174073:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941594.174077:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74800. 00000100:00000001:3.0:1773941594.174079:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294985767 00000400:00000200:0.0:1773941594.174081:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1773941594.174094:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294985770 00000400:00000200:0.0:1773941594.174095:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941594.174107:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acaa0 00000100:00000001:3.0:1773941594.174109:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294985767 00000400:00000010:0.0:1773941594.174111:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acaa0. 00000100:00000001:0.0:1773941594.174118:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941594.174123:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c3480 x1860112291495552/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941604 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941594.174129:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941594.174131:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c3480 x1860112291495552/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941604 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941594.174136:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941594.174137:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1773941594.174171:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941594.174176:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941594.174180:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941594.174184:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941594.174191:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941594.174194:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1773941594.174197:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941594.174198:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941594.174200:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941594.174204:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000001:0.0:1773941594.175931:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941594.175938:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941594.175940:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941594.175941:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941594.175945:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941594.175948:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941594.175952:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941594.175954:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941594.175957:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941594.176007:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324607e80 00000400:00000200:0.0:1773941594.176013:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941594.176025:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0x1119 [1] + 0 00000800:00000001:0.0:1773941594.176032:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941594.176038:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941594.176040:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941594.176042:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941594.176045:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941594.176048:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c3480 x1860112291495552/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941604 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941594.176055:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c3480 x1860112291495552/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941604 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941594.176069:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941594.176073:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941594.176085:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941594.176117:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941594.176121:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941594.176126:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941594.176128:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941594.176131:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941594.176136:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941594.176138:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941594.176141:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941594.176147:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c3480 x1860112291495552/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941604 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941594.176156:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c3480 x1860112291495552/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941604 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941594.176163:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941594.176164:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941594.176166:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941594.176168:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941594.176170:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941594.176171:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941594.176173:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941594.176179:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=95 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941594.176182:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941594.176184:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941594.176192:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941594.176195:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941594.176197:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941594.176199:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941594.176203:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c3480 x1860112291495552/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941604 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941594.176209:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941594.176211:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941594.176213:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941594.176215:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941594.176221:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941594.176223:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941594.176224:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941594.176226:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941594.176229:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941594.176230:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941594.176232:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941594.176236:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941594 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941594.176238:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941594.176244:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941594.176246:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941594.176249:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941594.176250:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941594.176252:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941594.176255:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941594.176257:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941594.176260:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941594.176263:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941594.176265:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941594.176267:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941594.176575:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941594.176579:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941594.176587:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c3480 x1860112291495552/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941604 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941594.176596:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291495552:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941594.176600:0:14798:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:2.0:1773941594.176602:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941594.176606:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941594.176613:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c3480 x1860112291495552/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941604 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1773941594.176616:0:14798:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:3.0:1773941594.176626:0:14798:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941594.176630:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941594.176632:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941594.176634:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03521c00. 02000000:00000001:2.0:1773941594.176637:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1773941594.176639:0:14798:0:(obd_class.h:1215:obd_import_event()) Process entered 00000020:00000001:2.0:1773941594.176640:0:2400:0:(genops.c:1236:class_import_put()) Process entered 10000000:01000000:3.0:1773941594.176643:0:14798:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000020:00000040:2.0:1773941594.176644:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000001:00000001:3.0:1773941594.176652:0:14798:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:2.0:1773941594.176653:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:3.0:1773941594.176665:0:14798:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000010:2.0:1773941594.176667:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d11188c00. 02000000:00000001:2.0:1773941594.176677:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000001:00000001:3.0:1773941594.176678:0:14798:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941594.176686:0:14798:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:2.0:1773941594.176687:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000001:00000001:3.0:1773941594.176693:0:14798:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000010:2.0:1773941594.176694:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c3480. 00000001:00000001:3.0:1773941594.176697:0:14798:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941594.176699:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 10000000:00000001:3.0:1773941594.176701:0:14798:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941594.176703:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941594.176705:0:14798:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941594.176706:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1773941594.176712:0:14798:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941594.176712:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941594.176718:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941594.176719:0:14798:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:3.0:1773941594.176723:0:14798:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941594.176725:0:14798:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1773941594.176727:0:14798:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:3.0:1773941594.176730:0:14798:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941594.176732:0:14798:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000100:00000001:2.0:1773941594.176739:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941594.176752:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:00000040:3.0:1773941594.176755:0:14798:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:3.0:1773941594.176772:0:14798:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:3.0:1773941594.176773:0:14798:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941594.176775:0:14798:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:3.0:1773941594.176778:0:14798:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:3.0:1773941594.176782:0:14798:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941594.176784:0:14798:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941594.176786:0:14798:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941594.176788:0:14798:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941594.176790:0:14798:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941594.179008:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d04dd4e00. 00010000:00000010:1.1:1773941594.179015:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d01488800. 00000020:00000001:2.0:1773941595.131219:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941595.131228:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941595.131234:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1773941595.451091:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294985770, at 4294985792 00000100:00000040:1.0:1773941595.451096:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294985770, at 4294985792 00000100:00000010:1.0:1773941595.451116:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f33c0. 00000100:00000010:0.0:1773941595.451121:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f35a0. 00000020:00000001:2.0:1773941596.155156:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941596.155165:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941596.155170:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941596.219117:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941596.219127:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941596.219132:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941596.219134:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941596.219136:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941596.219137:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941596.219140:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941596.219165:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941596.219168:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:1.0:1773941596.219171:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941596.219172:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941596.219181:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c0700. 00000020:00000040:1.0:1773941596.219186:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941596.219192:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941596.219195:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941596.219202:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941596.219210:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d181fd600. 00000100:00000001:1.0:1773941596.219218:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941596.219222:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941596.219225:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941596.219234:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d04dd4e00. 00010000:00000001:1.0:1773941596.219240:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941596.219244:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d01488800. 00000020:00000001:1.0:1773941596.219249:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941596.219251:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d01488800 with handle 0x8e86767622e2a8d to hash 00000020:00000001:1.0:1773941596.219253:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941596.219254:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941596.219257:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941596.219261:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941596.219264:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941596.219267:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a8d lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941596.219275:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a8d lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941596.219286:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a8d lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941596.219292:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941596.219293:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941596.219304:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941596.219306:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330528610752 : -113743180940864 : ffff988d1a1ab9c0) 00000100:00000001:1.0:1773941596.219308:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941596.219310:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00000100:00000001:1.0:1773941596.219315:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941596.219320:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941596.219323:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941596.219325:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941596.219336:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c0700 x1860112291495680/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941596.219343:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941596.219345:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941596.219349:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941596.219351:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941596.219353:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941596.219356:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291495680:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941596.219358:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941596.219364:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941596.219367:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941596.219382:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941596.219387:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d11188400. 02000000:00000001:1.0:1773941596.219388:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941596.219397:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941596.219400:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c5d8. 00000400:00000010:1.0:1773941596.219407:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000100:00000200:1.0:1773941596.219414:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291495680, portal 25 00000100:00000001:1.0:1773941596.219416:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941596.219417:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00000100:00000040:1.0:1773941596.219421:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c0700 x1860112291495680/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941612 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941596.219427:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941596.219430:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941596.219432:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de198. 00000100:00000200:1.0:1773941596.219438:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291495680, offset 0 00000400:00000200:1.0:1773941596.219444:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941596.219453:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941596.219461:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941596.219467:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941596.219471:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941596.219475:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941596.219477:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941596.219479:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[150, 149] 00000400:00000200:1.0:1773941596.219482:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:1.0:1773941596.219485:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [261:261:256:4294967295] 192.168.204.124@tcp LPNI seq info [151:151:8:4294967295] 00000400:00000200:1.0:1773941596.219489:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941596.219497:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:1.0:1773941596.219503:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:1.0:1773941596.219508:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfd400. 00000800:00000200:1.0:1773941596.219515:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:1.0:1773941596.219523:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:1.0:1773941596.219527:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfd400 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941596.219545:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941596.219549:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941596.219551:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294987815 00000100:00000010:1.0:1773941596.219555:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f33c0. 00000100:00000001:1.0:1773941596.219562:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1773941596.219570:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294987815 00000100:00000010:1.0:1773941596.219573:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3420. 00000100:00000001:1.0:1773941596.219576:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1773941596.219582:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941596.219585:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941596.219588:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941596.219590:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941596.219597:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 11 seconds 00000100:00000001:1.0:1773941596.219601:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941596.219604:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941596.219711:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941596.219721:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfd400. 00000400:00000200:0.0:1773941596.219728:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941596.219738:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941596.219745:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de198 00000400:00000010:0.0:1773941596.219749:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de198. 00000100:00000001:0.0:1773941596.219754:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941596.219762:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c0700 x1860112291495680/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941612 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941596.219776:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941596.219780:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d203c0700 x1860112291495680/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941612 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941596.219789:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941596.219791:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941596.220422:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941596.220434:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941596.220437:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941596.220450:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941596.220456:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941596.220460:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941596.220465:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941596.220468:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941596.220472:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941596.220475:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324607f00 00000400:00000200:0.0:1773941596.220478:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941596.220483:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0x1121 [1] + 224 00000800:00000001:0.0:1773941596.220489:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941596.220496:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941596.220497:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941596.220500:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941596.220503:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941596.220506:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c0700 x1860112291495680/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941612 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941596.220514:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d203c0700 x1860112291495680/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941612 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941596.220529:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941596.220535:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941596.220537:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941596.220592:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941596.220600:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941596.220602:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941596.220605:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:1.0:1773941596.220610:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:1.0:1773941596.220612:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:1.0:1773941596.220615:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941596.220621:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c0700 x1860112291495680/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941612 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941596.220630:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c0700 x1860112291495680/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941612 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941596.220636:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941596.220637:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941596.220639:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941596.220641:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941596.220643:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941596.220645:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941596.220647:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941596.220653:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=97 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941596.220655:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941596.220658:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941596.220661:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:1.0:1773941596.220663:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941596.220666:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941596.220668:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941596.220672:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941596.220676:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941596.220678:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941596.220680:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941596.220688:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941596.220691:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941596.220695:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d203c0700 x1860112291495680/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941612 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941596.220701:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941596.220702:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941596.220703:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941596.220706:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941596.220708:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941596.220709:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941596.220710:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941596.220711:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941596.220712:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941596.220713:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941596.220714:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941596.220717:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941596.220719:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941596.220721:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941596.220723:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 317 00000400:00000200:1.0:1773941596.220727:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941596.220731:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941596.220734:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941596.220737:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941596.220739:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941596.220741:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941596.220744:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941596.220746:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941596.220748:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941596.220750:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941596.220751:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941596.220753:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941596.220755:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941596.220756:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941596.220757:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941596.220763:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941596.220766:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941596.220768:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941596.220771:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941596.220773:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941596.220776:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941596.220777:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941596.220780:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941596.220781:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941596.220784:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c3b80. 00000020:00000040:1.0:1773941596.220787:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941596.220789:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941596.220808:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941596.220810:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941596.220812:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941596.220814:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941596.220817:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d11188000. 00000100:00000001:1.0:1773941596.220820:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941596.220825:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c3b80 x1860112291495808/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941596.220832:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941596.220834:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941596.220837:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000e15b8eeb] to pc [ptlrpcd_rcv+-1] req@ffff988d203c3b80 x1860112291495808/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941596.220857:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941596.220860:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941596.220861:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941596.220864:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941596.220866:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941596.220867:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941596.220869:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941596.220870:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941596.220872:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941596.220879:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c0700 x1860112291495680/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941612 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941596.220885:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941596.220888:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941596.220890:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941596.220892:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941596.220895:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c0700 x1860112291495680/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941612 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941596.220901:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291495680:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941596.220905:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941596.220907:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941596.220908:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941596.220909:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941596.220912:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941596.220915:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1773941596.220916:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c0700 x1860112291495680/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941612 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941596.220920:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941596.220926:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941596.220930:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1a1ab9c0. 00000100:00000040:2.0:1773941596.220932:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c3b80 x1860112291495808/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941596.220934:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941596.220937:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:1.0:1773941596.220941:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941596.220945:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941596.220948:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1773941596.220952:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000020:00000040:1.0:1773941596.220952:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488800 refcount=4 00000100:00000001:2.0:1773941596.220956:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941596.220957:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941596.220981:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00010000:1.0:1773941596.220985:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a8d lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941596.220992:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00010000:00010000:1.0:1773941596.220996:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a8d lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941596.221006:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 02000000:00000001:2.0:1773941596.221008:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00010000:00010000:1.0:1773941596.221010:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a8d lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941596.221011:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941596.221017:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941596.221024:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00100000:2.0:1773941596.221025:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291495808:192.168.204.114@tcp:250:ll_cfg_requeue.0 00010000:00000001:1.0:1773941596.221027:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941596.221031:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a8d lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941596.221032:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941596.221036:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941596.221039:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941596.221040:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 02000000:00000001:2.0:1773941596.221042:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:1.0:1773941596.221043:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941596.221047:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 02000000:00000010:2.0:1773941596.221049:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a7c00. 00010000:00010000:1.0:1773941596.221050:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a8d lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941596.221054:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941596.221062:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:1.0:1773941596.221062:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d01488800 already unused, calling callback (ffffffffc12b4c20) 10000000:00000001:1.0:1773941596.221067:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000010:2.0:1773941596.221069:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 10000000:00010000:1.0:1773941596.221071:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a8d lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941596.221075:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00010000:00000001:1.0:1773941596.221078:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000100:00000200:2.0:1773941596.221081:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291495808, portal 25 00010000:00000001:1.0:1773941596.221081:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941596.221084:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1773941596.221086:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000020:00000040:1.0:1773941596.221087:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488800 refcount=5 00000100:00000001:2.0:1773941596.221089:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631469952 : -113743078081664 : ffff988d203c3b80) 00000020:00000001:1.0:1773941596.221090:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941596.221094:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00000100:00000040:2.0:1773941596.221097:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c3b80 x1860112291495808/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941606 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941596.221099:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941596.221103:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a8d lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941596.221117:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 10000000:00000001:1.0:1773941596.221119:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941596.221127:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a8d lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941596.221135:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 10000000:01000000:1.0:1773941596.221137:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000010:2.0:1773941596.221141:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045167f8. 10000000:00000040:1.0:1773941596.221142:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941596.221144:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941596.221149:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000100:00000200:2.0:1773941596.221151:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291495808, offset 0 00010000:00000001:1.0:1773941596.221151:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:1.0:1773941596.221156:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941596.221158:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:1.0:1773941596.221160:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941596.221162:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941596.221163:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000020:00000040:1.0:1773941596.221166:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d01488800 with handle 0x8e86767622e2a8d from hash 00010000:00000001:1.0:1773941596.221170:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941596.221171:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:2.0:1773941596.221175:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941596.221175:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941596.221179:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941596.221183:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000400:00000200:2.0:1773941596.221185:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:1.0:1773941596.221187:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000400:00000200:2.0:1773941596.221192:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:1.0:1773941596.221194:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941596.221198:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:2.0:1773941596.221200:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00000001:1.0:1773941596.221202:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941596.221206:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 10000000:00000001:1.0:1773941596.221208:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941596.221213:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00010000:1.0:1773941596.221215:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a8d lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941596.221218:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[150, 151] 00000400:00000200:2.0:1773941596.221228:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:2.0:1773941596.221236:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [262:262:256:4294967295] 192.168.204.114@tcp LPNI seq info [152:152:8:4294967295] 00010000:00000001:1.0:1773941596.221237:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941596.221240:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941596.221243:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000400:00000200:2.0:1773941596.221248:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941596.221248:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941596.221252:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941596.221255:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941596.221257:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941596.221261:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a8d lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941596.221265:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00010000:00000040:1.0:1773941596.221279:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d04dd4e00 count: 0 00000800:00000200:2.0:1773941596.221280:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:2.0:1773941596.221287:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0db900. 00000800:00000200:2.0:1773941596.221296:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00010000:00000001:1.0:1773941596.221303:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000200:2.0:1773941596.221308:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00010000:00000001:1.0:1773941596.221309:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000800:00000200:2.0:1773941596.221315:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0db900 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:1.0:1773941596.221316:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941596.221324:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c0700 x1860112291495680/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941612 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941596.221330:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941596.221333:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1773941596.221334:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941596.221336:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d11188400. 02000000:00000001:1.0:1773941596.221341:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000400:00000200:2.0:1773941596.221343:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000020:00000001:1.0:1773941596.221345:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941596.221348:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941596.221350:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294987817 00000020:00000001:1.0:1773941596.221352:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941596.221357:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d181fd600. 02000000:00000001:1.0:1773941596.221361:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1773941596.221363:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294987815 02000000:00000001:1.0:1773941596.221363:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941596.221366:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c0700. 00000100:00000001:2.0:1773941596.221386:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294987817 00000100:00000001:1.0:1773941596.221389:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941596.221392:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941596.221393:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294987815 10000000:00000001:1.0:1773941596.221394:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941596.221400:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000100:00000001:2.0:1773941596.221402:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 10000000:00000001:1.0:1773941596.221404:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000100:00000001:2.0:1773941596.221409:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 10000000:00000010:1.0:1773941596.221409:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d04d67a00. 00000100:00000001:2.0:1773941596.221416:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941596.221416:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d181fd600. 00000100:00000001:2.0:1773941596.221423:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:1.0:1773941596.221424:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d181fce00. 00000040:00000010:1.0:1773941596.221429:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d181fc800. 00000020:00000010:1.0:1773941596.221435:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d18b18000. 00000100:00000001:2.0:1773941596.221441:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000020:00000010:1.0:1773941596.221441:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d181fc200. 00000100:00000001:2.0:1773941596.221446:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 80000000:00000010:1.0:1773941596.221448:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d054f3a20. 00000100:00000001:2.0:1773941596.221451:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00040000:00000010:1.0:1773941596.221454:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d181fc400. 00000100:00000001:2.0:1773941596.221455:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000800:00000200:0.0:1773941596.221459:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1773941596.221460:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 20000000:00000010:1.0:1773941596.221461:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d181fd800. 00000800:00000010:0.0:1773941596.221465:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0db900. 00000100:00000001:2.0:1773941596.221468:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:00000040:1.0:1773941596.221470:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 00000400:00000200:0.0:1773941596.221471:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 10000000:00000001:1.0:1773941596.221474:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000200:0.0:1773941596.221480:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000020:00000010:1.0:1773941596.221481:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d181fce00. 00000400:00000200:0.0:1773941596.221487:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045167f8 00000040:00000010:1.0:1773941596.221489:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d181fc800. 00000400:00000010:0.0:1773941596.221490:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045167f8. 00000100:00000001:0.0:1773941596.221495:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000020:00000010:1.0:1773941596.221501:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d18b18000. 00000100:00000200:0.0:1773941596.221502:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c3b80 x1860112291495808/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941606 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1773941596.221507:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d181fc200. 80000000:00000010:1.0:1773941596.221512:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d054f3a20. 00000100:00000001:0.0:1773941596.221514:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00040000:00000010:1.0:1773941596.221517:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d181fc400. 00000100:00000040:0.0:1773941596.221520:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c3b80 x1860112291495808/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941606 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 20000000:00000010:1.0:1773941596.221522:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d181fd800. 00000020:00000010:1.0:1773941596.221526:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d181fd600. 10000000:00000010:1.0:1773941596.221529:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d04d67a00. 10000000:01000000:1.0:1773941596.221534:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941596.221541:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941596.221544:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 10000000:00020000:1.0:1773941596.221545:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 00000100:00000001:0.0:1773941596.221549:0:2393:0:(events.c:68:request_out_callback()) Process leaving 10000000:00000001:1.0:1773941596.221550:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941596.221555:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941596.221557:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941596.221560:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941596.222706:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941596.222716:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941596.222718:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941596.222720:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941596.222724:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941596.222728:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941596.222733:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941596.222736:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941596.222739:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941596.222742:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324607f80 00000400:00000200:0.0:1773941596.222745:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941596.222749:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0x1131 [1] + 0 00000800:00000001:0.0:1773941596.222755:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941596.222772:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941596.222774:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941596.222780:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941596.222783:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941596.222787:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c3b80 x1860112291495808/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941606 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941596.222796:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c3b80 x1860112291495808/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941606 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941596.222810:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941596.222816:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941596.222819:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941596.222842:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941596.222845:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941596.222849:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941596.222850:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941596.222852:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941596.222855:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941596.222857:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941596.222859:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941596.222864:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c3b80 x1860112291495808/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941606 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941596.222872:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c3b80 x1860112291495808/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941606 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941596.222878:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941596.222879:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941596.222881:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941596.222882:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941596.222883:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941596.222885:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941596.222886:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941596.222889:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=97 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941596.222891:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941596.222893:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941596.222895:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941596.222897:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941596.222899:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941596.222900:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941596.222903:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c3b80 x1860112291495808/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941606 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941596.222908:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941596.222910:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941596.222917:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941596.222919:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941596.222923:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941596.222925:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941596.222926:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941596.222928:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941596.222929:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941596.222931:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941596.222933:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:2.0:1773941596.222936:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941596 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941596.222938:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941596.222943:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:2.0:1773941596.222945:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941596.222946:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941596.222948:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941596.222949:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941596.222952:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:2.0:1773941596.222954:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941596.222956:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941596.222975:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941596.222979:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:2.0:1773941596.222981:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941596.223237:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941596.223245:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941596.223257:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c3b80 x1860112291495808/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941606 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941596.223257:0:14799:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00080000:2.0:1773941596.223271:0:14799:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:2.0:1773941596.223283:0:14799:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00100000:3.0:1773941596.223290:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c3b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291495808:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941596.223303:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941596.223307:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941596.223322:0:14799:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000040:3.0:1773941596.223323:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c3b80 x1860112291495808/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941606 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:01000000:2.0:1773941596.223328:0:14799:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:2.0:1773941596.223337:0:14799:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:3.0:1773941596.223341:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:2.0:1773941596.223343:0:14799:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:3.0:1773941596.223347:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:2.0:1773941596.223351:0:14799:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941596.223354:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a7c00. 00000001:00000001:2.0:1773941596.223356:0:14799:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941596.223360:0:14799:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:3.0:1773941596.223361:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:2.0:1773941596.223363:0:14799:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941596.223366:0:2400:0:(genops.c:1236:class_import_put()) Process entered 10000000:00000001:2.0:1773941596.223384:0:14799:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1773941596.223386:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941596.223390:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941596.223392:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d11188000. 00000100:00000001:2.0:1773941596.223401:0:14799:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000001:3.0:1773941596.223404:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941596.223408:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:2.0:1773941596.223410:0:14799:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000010:3.0:1773941596.223411:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c3b80. 00000100:00000001:2.0:1773941596.223420:0:14799:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:3.0:1773941596.223421:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941596.223423:0:14799:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941596.223426:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941596.223427:0:14799:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941596.223439:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1773941596.223439:0:14799:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941596.223444:0:14799:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941596.223447:0:14799:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:2.0:1773941596.223449:0:14799:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00000001:3.0:1773941596.223452:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941596.223455:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941596.223459:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941596.223462:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1773941596.223462:0:14799:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:2.0:1773941596.223464:0:14799:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941596.223467:0:14799:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:2.0:1773941596.223472:0:14799:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:2.0:1773941596.223477:0:14799:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941596.223480:0:14799:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941596.223482:0:14799:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941596.223484:0:14799:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941596.223486:0:14799:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941596.227010:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d04dd4e00. 00010000:00000010:1.1:1773941596.227017:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d01488800. 00000020:00000001:2.0:1773941597.179210:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941597.179218:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941597.179223:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:2.0:1773941597.435234:0:8:0:(pinger.c:229:ptlrpc_pinger_process_import()) c1c1ff21-5f68-4bf6-bdc4-258644b24465->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1773941597.435248:0:8:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:2.0:1773941597.435265:0:8:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06506680. 00000020:00000040:2.0:1773941597.435274:0:8:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941597.435281:0:8:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941597.435285:0:8:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941597.435293:0:8:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941597.435304:0:8:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff988d0a0dbd00. 00000100:00000001:2.0:1773941597.435314:0:8:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941597.435331:0:8:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging c1c1ff21-5f68-4bf6-bdc4-258644b24465->MGS req@ffff988d06506680 x1860112291495936/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941597.435381:0:8:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941597.435396:0:8:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941597.435432:0:8:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000022f6a110] to pc [ptlrpcd_01_00+0] req@ffff988d06506680 x1860112291495936/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941597.435485:0:8:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941597.435494:0:8:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (325) 00000100:00000001:2.0:1773941597.435522:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941597.435526:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:2.0:1773941597.435528:0:2404:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:2.0:1773941597.435535:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941597.435538:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941597.435547:0:2404:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941597.435552:0:2404:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941597.435555:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941597.435559:0:2404:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941597.435563:0:2404:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941597.435571:0:2404:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06506680 x1860112291495936/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941597.435583:0:2404:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941597.435586:0:2404:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941597.435591:0:2404:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941597.435595:0:2404:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941597.435598:0:2404:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941597.435602:0:2404:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06506680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2404:1860112291495936:192.168.204.114@tcp:400:kworker.0 00000100:00000001:2.0:1773941597.435608:0:2404:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941597.435615:0:2404:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941597.435619:0:2404:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941597.435622:0:2404:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:2.0:1773941597.435628:0:2404:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff988d03356200. 02000000:00000001:2.0:1773941597.435631:0:2404:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941597.435642:0:2404:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941597.435647:0:2404:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000400:00000010:2.0:1773941597.435667:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000200:2.0:1773941597.435681:0:2404:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112291495936, portal 25 00000100:00000001:2.0:1773941597.435685:0:2404:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941597.435689:0:2404:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584064 : -113743512967552 : ffff988d06506680) 00000100:00000040:2.0:1773941597.435698:0:2404:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06506680 x1860112291495936/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941613 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941597.435759:0:2404:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941597.435765:0:2404:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:2.0:1773941597.435770:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516660. 00000100:00000200:2.0:1773941597.435776:0:2404:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1860112291495936, offset 0 00000400:00000200:2.0:1773941597.435800:0:2404:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941597.435829:0:2404:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941597.435842:0:2404:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941597.435849:0:2404:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941597.435854:0:2404:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941597.435859:0:2404:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941597.435863:0:2404:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941597.435867:0:2404:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[152, 151] 00000400:00000200:2.0:1773941597.435871:0:2404:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:2.0:1773941597.435877:0:2404:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [263:263:256:4294967295] 192.168.204.124@tcp LPNI seq info [153:153:8:4294967295] 00000400:00000200:2.0:1773941597.435882:0:2404:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:2.0:1773941597.435896:0:2404:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:2.0:1773941597.435904:0:2404:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:2.0:1773941597.435911:0:2404:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0dbb00. 00000800:00000200:2.0:1773941597.435924:0:2404:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:2.0:1773941597.435938:0:2404:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:2.0:1773941597.435946:0:2404:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0dbb00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:2.0:1773941597.436075:0:2404:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941597.436085:0:2404:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941597.436090:0:2404:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294989032 00000100:00000001:2.0:1773941597.436122:0:2404:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294987817 00000100:00000001:2.0:1773941597.436131:0:2404:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294989032 00000100:00000001:2.0:1773941597.436138:0:2404:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294987817 00000100:00000001:2.0:1773941597.436144:0:2404:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941597.436149:0:2404:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941597.436166:0:2404:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941597.436172:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941597.436296:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941597.436309:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0dbb00. 00000400:00000200:0.0:1773941597.436321:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941597.436374:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941597.436386:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516660 00000400:00000010:0.0:1773941597.436423:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516660. 00000100:00000001:0.0:1773941597.436429:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941597.436438:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06506680 x1860112291495936/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941613 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941597.436450:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941597.436453:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06506680 x1860112291495936/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941613 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941597.436461:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941597.436462:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941597.437707:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941597.437721:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941597.437724:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941597.437726:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941597.437733:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941597.437739:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941597.437777:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941597.437782:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941597.437789:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941597.437794:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324608000 00000400:00000200:0.0:1773941597.437800:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941597.437807:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0x1139 [1] + 224 00000800:00000001:0.0:1773941597.437843:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941597.437850:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941597.437852:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941597.437855:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941597.437860:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941597.437865:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06506680 x1860112291495936/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941613 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941597.437877:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff988d06506680 x1860112291495936/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941613 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941597.437896:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941597.437904:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941597.437915:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941597.437995:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941597.438017:0:2404:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941597.438026:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941597.438029:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941597.438034:0:2404:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941597.438043:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941597.438047:0:2404:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941597.438052:0:2404:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941597.438063:0:2404:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06506680 x1860112291495936/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941613 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941597.438080:0:2404:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06506680 x1860112291495936/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941613 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941597.438121:0:2404:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941597.438123:0:2404:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941597.438128:0:2404:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941597.438133:0:2404:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941597.438137:0:2404:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941597.438140:0:2404:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941597.438145:0:2404:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941597.438156:0:2404:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=98 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941597.438160:0:2404:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941597.438163:0:2404:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941597.438167:0:2404:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941597.438171:0:2404:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:2.0:1773941597.438175:0:2404:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:2.0:1773941597.438181:0:2404:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:2.0:1773941597.438186:0:2404:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:2.0:1773941597.438191:0:2404:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941597.438194:0:2404:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941597.438196:0:2404:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941597.438199:0:2404:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:2.0:1773941597.438224:0:2404:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d06506680 x1860112291495936/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941613 ref 1 fl Rpc:RNQU/200/0 rc 0/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941597.438234:0:2404:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941597.438236:0:2404:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941597.438239:0:2404:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941597.438244:0:2404:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:2.0:1773941597.438248:0:2404:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:2.0:1773941597.438250:0:2404:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941597.438252:0:2404:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941597.438253:0:2404:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941597.438255:0:2404:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941597.438257:0:2404:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941597.438258:0:2404:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941597.438261:0:2404:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1773941597.438264:0:2404:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1773941597.438267:0:2404:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:2.0:1773941597.438270:0:2404:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 319 00000400:00000200:2.0:1773941597.438276:0:2404:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941597.438281:0:2404:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:2.0:1773941597.438286:0:2404:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941597.438289:0:2404:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941597.438292:0:2404:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941597.438294:0:2404:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:2.0:1773941597.438299:0:2404:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941597.438301:0:2404:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941597.438304:0:2404:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941597.438308:0:2404:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941597.438311:0:2404:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941597.438314:0:2404:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:2.0:1773941597.438317:0:2404:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941597.438319:0:2404:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:2.0:1773941597.438321:0:2404:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941597.438323:0:2404:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:2.0:1773941597.438332:0:2404:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:2.0:1773941597.438337:0:2404:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:2.0:1773941597.438340:0:2404:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941597.438343:0:2404:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1773941597.438367:0:2404:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1773941597.438372:0:2404:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941597.438375:0:2404:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941597.438378:0:2404:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941597.438382:0:2404:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06506a00. 00000020:00000040:2.0:1773941597.438386:0:2404:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:2.0:1773941597.438390:0:2404:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1773941597.438392:0:2404:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941597.438396:0:2404:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941597.438398:0:2404:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941597.438401:0:2404:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941597.438407:0:2404:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d035a5000. 00000100:00000001:2.0:1773941597.438411:0:2404:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1773941597.438419:0:2404:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d06506a00 x1860112291496064/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941597.438428:0:2404:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941597.438432:0:2404:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941597.438436:0:2404:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6b603d8] to pc [ptlrpcd_rcv+-1] req@ffff988d06506a00 x1860112291496064/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941597.438475:0:2404:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941597.438478:0:2404:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:2.0:1773941597.438480:0:2404:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:2.0:1773941597.438486:0:2404:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06506680 x1860112291495936/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941613 ref 1 fl Rpc:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941597.438495:0:2404:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941597.438498:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941597.438501:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941597.438512:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:2.0:1773941597.438514:0:2404:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06506680 x1860112291495936/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941613 ref 1 fl Interpret:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941597.438518:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941597.438526:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00100000:2.0:1773941597.438537:0:2404:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06506680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2404:1860112291495936:192.168.204.114@tcp:400:kworker.0 00000100:00000040:3.0:1773941597.438545:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06506a00 x1860112291496064/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941597.438551:0:2404:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941597.438563:0:2404:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1773941597.438574:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:2.0:1773941597.438576:0:2404:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06506680 x1860112291495936/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941613 ref 1 fl Complete:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941597.438578:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941597.438596:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1773941597.438597:0:2404:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941597.438606:0:2404:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1773941597.438608:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000010:2.0:1773941597.438611:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff988d03356200. 02000000:00000001:2.0:1773941597.438623:0:2404:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:3.0:1773941597.438624:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941597.438627:0:2404:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941597.438632:0:2404:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941597.438637:0:2404:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00100000:3.0:1773941597.438640:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291496064:192.168.204.114@tcp:250:ptlrpcd_01_01.0 02000000:00000010:2.0:1773941597.438641:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff988d0a0dbd00. 02000000:00000001:2.0:1773941597.438650:0:2404:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941597.438656:0:2404:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1773941597.438658:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000010:2.0:1773941597.438660:0:2404:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06506680. 00000100:00000001:2.0:1773941597.438666:0:2404:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 02000000:00000001:3.0:1773941597.438668:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1773941597.438669:0:2404:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941597.438672:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941597.438676:0:2404:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941597.438678:0:2404:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941597.438684:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1773941597.438688:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941597.438690:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941597.438694:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941597.438703:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03522800. 02000000:00000001:3.0:1773941597.438706:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941597.438712:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941597.438716:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00000400:00000010:3.0:1773941597.438723:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00000100:00000200:3.0:1773941597.438730:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291496064, portal 25 00000100:00000001:3.0:1773941597.438733:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941597.438734:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584960 : -113743512966656 : ffff988d06506a00) 00000100:00000040:3.0:1773941597.438742:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06506a00 x1860112291496064/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941607 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941597.438750:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941597.438752:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:3.0:1773941597.438755:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ace58. 00000100:00000200:3.0:1773941597.438757:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291496064, offset 0 00000400:00000200:3.0:1773941597.438763:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941597.438768:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941597.438775:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941597.438780:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941597.438783:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941597.438788:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941597.438791:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941597.438807:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[152, 153] 00000400:00000200:3.0:1773941597.438813:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:3.0:1773941597.438819:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [264:264:256:4294967295] 192.168.204.114@tcp LPNI seq info [154:154:8:4294967295] 00000400:00000200:3.0:1773941597.438825:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941597.438837:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:3.0:1773941597.438845:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:3.0:1773941597.438851:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74c00. 00000800:00000200:3.0:1773941597.438858:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:3.0:1773941597.438879:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:3.0:1773941597.438894:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74c00 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:3.0:1773941597.438914:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941597.438921:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:3.0:1773941597.438924:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294989034 00000100:00000001:3.0:1773941597.438936:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294989032 00000100:00000001:3.0:1773941597.438954:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294989034 00000100:00000001:3.0:1773941597.439032:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294989032 00000100:00000001:3.0:1773941597.439039:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941597.439043:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941597.439048:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941597.439069:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941597.439078:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941597.439081:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1773941597.439085:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941597.439087:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941597.439090:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941597.439098:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1773941597.439105:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:0.0:1773941597.439109:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74c00. 00000400:00000200:0.0:1773941597.439119:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941597.439128:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941597.439133:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ace58 00000400:00000010:0.0:1773941597.439136:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ace58. 00000100:00000001:0.0:1773941597.439141:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941597.439165:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06506a00 x1860112291496064/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941607 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941597.439188:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941597.439192:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06506a00 x1860112291496064/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941607 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941597.439201:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941597.439204:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941597.441225:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941597.441234:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941597.441236:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941597.441239:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941597.441243:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941597.441248:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941597.441254:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941597.441258:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941597.441262:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941597.441266:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324608080 00000400:00000200:0.0:1773941597.441269:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941597.441273:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0x1141 [1] + 0 00000800:00000001:0.0:1773941597.441278:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941597.441284:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941597.441286:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941597.441290:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941597.441294:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941597.441307:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06506a00 x1860112291496064/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941607 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941597.441327:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d06506a00 x1860112291496064/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941607 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941597.441362:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941597.441370:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941597.441373:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941597.441412:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941597.441416:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941597.441422:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941597.441424:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941597.441426:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941597.441431:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941597.441433:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941597.441436:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941597.441442:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06506a00 x1860112291496064/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941607 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941597.441452:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06506a00 x1860112291496064/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941607 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941597.441459:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941597.441461:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941597.441463:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941597.441466:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941597.441468:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941597.441478:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941597.441481:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941597.441488:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=98 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941597.441492:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941597.441495:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941597.441497:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941597.441501:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941597.441504:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941597.441508:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941597.441515:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06506a00 x1860112291496064/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941607 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941597.441525:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941597.441529:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941597.441532:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941597.441535:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941597.441543:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941597.441547:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941597.441548:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941597.441551:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941597.441554:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941597.441556:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941597.441559:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941597.441562:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941597 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941597.441565:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941597.441572:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941597.441574:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941597.441577:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941597.441579:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941597.441581:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941597.441585:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941597.441587:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941597.441589:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941597.441593:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941597.441604:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941597.441606:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941597.442037:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941597.442057:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941597.442068:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06506a00 x1860112291496064/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941607 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1773941597.442083:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291496064:192.168.204.114@tcp:250:ptlrpcd_01_01.0 00000100:00000001:2.0:1773941597.442093:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941597.442098:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1773941597.442111:0:14800:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000040:2.0:1773941597.442116:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06506a00 x1860112291496064/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941607 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ptlrpcd_01_01.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1773941597.442147:0:14800:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00000001:2.0:1773941597.442149:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1773941597.442154:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00100000:3.0:1773941597.442160:0:14800:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000010:2.0:1773941597.442160:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03522800. 00000100:00000001:3.0:1773941597.442171:0:14800:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:2.0:1773941597.442172:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 10000000:01000000:3.0:1773941597.442178:0:14800:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000020:00000001:2.0:1773941597.442179:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941597.442184:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000001:00000001:3.0:1773941597.442187:0:14800:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:2.0:1773941597.442189:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:2.0:1773941597.442193:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d035a5000. 00000001:00000001:3.0:1773941597.442194:0:14800:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941597.442199:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000001:00000001:3.0:1773941597.442202:0:14800:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941597.442203:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000001:00000001:3.0:1773941597.442208:0:14800:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000010:2.0:1773941597.442209:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06506a00. 00000001:00000001:3.0:1773941597.442213:0:14800:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:2.0:1773941597.442216:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000001:00000001:3.0:1773941597.442218:0:14800:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941597.442221:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:3.0:1773941597.442223:0:14800:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941597.442227:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941597.442228:0:14800:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941597.442237:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1773941597.442239:0:14800:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941597.442242:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941597.442245:0:14800:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941597.442248:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941597.442250:0:14800:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:2.0:1773941597.442253:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941597.442255:0:14800:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1773941597.442260:0:14800:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:3.0:1773941597.442263:0:14800:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941597.442266:0:14800:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:3.0:1773941597.442269:0:14800:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:3.0:1773941597.442303:0:14800:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:3.0:1773941597.442307:0:14800:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941597.442311:0:14800:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:3.0:1773941597.442317:0:14800:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:3.0:1773941597.442325:0:14800:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941597.442330:0:14800:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941597.442333:0:14800:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:3.0:1773941597.442338:0:14800:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941597.442342:0:14800:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941597.907071:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941597.907079:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941597.907084:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941597.907087:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941597.907088:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941597.907090:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941597.907093:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941597.907095:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941597.907098:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:1.0:1773941597.907101:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941597.907103:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941597.907114:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c0700. 00000020:00000040:1.0:1773941597.907120:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941597.907126:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941597.907129:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941597.907134:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941597.907142:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d181fc400. 00000100:00000001:1.0:1773941597.907149:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941597.907154:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941597.907157:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941597.907165:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d04dd4e00. 00010000:00000001:1.0:1773941597.907172:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941597.907190:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d01488800. 00000020:00000001:1.0:1773941597.907194:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941597.907196:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d01488800 with handle 0x8e86767622e2a94 to hash 00000020:00000001:1.0:1773941597.907198:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941597.907199:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941597.907201:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941597.907205:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941597.907207:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941597.907211:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a94 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941597.907218:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a94 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941597.907228:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a94 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941597.907232:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941597.907234:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941597.907239:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941597.907240:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330528610752 : -113743180940864 : ffff988d1a1ab9c0) 00000100:00000001:1.0:1773941597.907242:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941597.907243:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00000100:00000001:1.0:1773941597.907247:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941597.907250:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941597.907252:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941597.907254:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941597.907262:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c0700 x1860112291496192/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941597.907279:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941597.907280:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941597.907283:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941597.907286:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941597.907291:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941597.907294:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291496192:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941597.907306:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941597.907313:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941597.907315:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941597.907317:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941597.907321:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d11188400. 02000000:00000001:1.0:1773941597.907322:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941597.907328:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941597.907331:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c5d8. 00000400:00000010:1.0:1773941597.907336:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000100:00000200:1.0:1773941597.907353:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291496192, portal 25 00000100:00000001:1.0:1773941597.907355:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941597.907356:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00000100:00000040:1.0:1773941597.907360:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c0700 x1860112291496192/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941613 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941597.907375:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941597.907378:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941597.907379:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113ded48. 00000100:00000200:1.0:1773941597.907381:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291496192, offset 0 00000400:00000200:1.0:1773941597.907386:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941597.907393:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941597.907399:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941597.907402:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941597.907406:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941597.907409:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941597.907411:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941597.907413:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[154, 153] 00000400:00000200:1.0:1773941597.907416:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:1.0:1773941597.907419:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [265:265:256:4294967295] 192.168.204.124@tcp LPNI seq info [155:155:8:4294967295] 00000400:00000200:1.0:1773941597.907421:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941597.907429:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:1.0:1773941597.907438:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:1.0:1773941597.907442:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfdd00. 00000800:00000200:1.0:1773941597.907447:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:1.0:1773941597.907453:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:1.0:1773941597.907456:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfdd00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941597.907473:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941597.907477:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941597.907480:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294989503 00000100:00000001:1.0:1773941597.907486:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294989034 00000100:00000001:1.0:1773941597.907489:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294989503 00000100:00000001:1.0:1773941597.907492:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294989034 00000100:00000001:1.0:1773941597.907494:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941597.907496:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941597.907497:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941597.907499:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941597.907501:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 11 seconds 00000100:00000001:1.0:1773941597.907504:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941597.907507:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941597.907625:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941597.907631:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfdd00. 00000400:00000200:0.0:1773941597.907637:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941597.907644:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941597.907650:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113ded48 00000400:00000010:0.0:1773941597.907653:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113ded48. 00000100:00000001:0.0:1773941597.907657:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941597.907662:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c0700 x1860112291496192/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941613 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941597.907673:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941597.907677:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d203c0700 x1860112291496192/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941613 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941597.907685:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941597.907687:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941597.908387:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941597.908407:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941597.908412:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941597.908415:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941597.908425:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941597.908433:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941597.908443:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941597.908448:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941597.908454:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941597.908458:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324608100 00000400:00000200:0.0:1773941597.908462:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941597.908467:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0x1149 [1] + 224 00000800:00000001:0.0:1773941597.908488:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941597.908497:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941597.908499:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941597.908503:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941597.908508:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941597.908516:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c0700 x1860112291496192/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941613 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941597.908525:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d203c0700 x1860112291496192/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941613 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941597.908545:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941597.908553:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941597.908561:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941597.908595:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941597.908604:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941597.908606:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941597.908609:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:1.0:1773941597.908613:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:1.0:1773941597.908615:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:1.0:1773941597.908618:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941597.908623:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c0700 x1860112291496192/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941613 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941597.908638:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c0700 x1860112291496192/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941613 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941597.908644:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941597.908645:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941597.908654:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941597.908657:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941597.908659:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941597.908662:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941597.908664:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941597.908671:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=98 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941597.908674:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=0 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941597.908676:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941597.908680:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:1.0:1773941597.908682:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941597.908685:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941597.908687:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941597.908691:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941597.908695:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941597.908698:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941597.908700:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941597.908702:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941597.908703:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941597.908706:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d203c0700 x1860112291496192/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941613 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941597.908712:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941597.908713:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941597.908715:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941597.908718:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941597.908720:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941597.908721:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941597.908723:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941597.908723:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941597.908724:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941597.908726:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941597.908727:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941597.908729:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941597.908730:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941597.908733:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941597.908739:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 320 00000400:00000200:1.0:1773941597.908743:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941597.908747:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941597.908751:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941597.908754:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941597.908756:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941597.908758:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941597.908761:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941597.908763:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941597.908766:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941597.908768:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941597.908769:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941597.908771:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941597.908773:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941597.908774:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941597.908775:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941597.908777:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941597.908779:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941597.908781:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941597.908783:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941597.908785:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941597.908788:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941597.908790:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941597.908793:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941597.908794:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941597.908799:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c1880. 00000020:00000040:1.0:1773941597.908801:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941597.908805:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941597.908806:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941597.908809:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941597.908811:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941597.908816:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941597.908827:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d11189000. 00000100:00000001:1.0:1773941597.908832:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941597.908840:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c1880 x1860112291496320/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941597.908851:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941597.908855:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941597.908861:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000008b2304da] to pc [ptlrpcd_rcv+-1] req@ffff988d203c1880 x1860112291496320/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941597.908889:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941597.908893:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941597.908895:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941597.908898:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941597.908901:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941597.908903:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941597.908906:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941597.908909:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941597.908911:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941597.908918:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c0700 x1860112291496192/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941613 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941597.908927:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:2.0:1773941597.908930:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941597.908930:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941597.908932:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941597.908935:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941597.908936:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941597.908940:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941597.908944:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c0700 x1860112291496192/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941613 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941597.908949:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c1880 x1860112291496320/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941597.908984:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291496192:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941597.908990:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941597.908993:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941597.908995:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941597.908996:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941597.909005:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c0700 x1860112291496192/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941613 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941597.909011:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941597.909014:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941597.909016:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941597.909019:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:1.0:1773941597.909026:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000001:2.0:1773941597.909027:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941597.909031:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941597.909032:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941597.909034:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:2.0:1773941597.909035:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:1.0:1773941597.909036:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488800 refcount=4 02000000:00000001:2.0:1773941597.909038:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1773941597.909040:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 02000000:00000001:2.0:1773941597.909042:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941597.909045:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00000100:00100000:2.0:1773941597.909048:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291496320:192.168.204.114@tcp:250:ll_cfg_requeue.0 00010000:00010000:1.0:1773941597.909052:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a94 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941597.909053:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941597.909058:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941597.909061:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941597.909063:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 02000000:00000001:2.0:1773941597.909064:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00010000:1.0:1773941597.909069:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a94 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941597.909077:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941597.909079:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a94 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941597.909084:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941597.909085:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941597.909087:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a94 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941597.909091:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941597.909093:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941597.909096:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 02000000:00000010:2.0:1773941597.909098:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a5000. 00010000:00010000:1.0:1773941597.909099:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a94 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941597.909102:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941597.909106:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941597.909111:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000400:00000010:2.0:1773941597.909117:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00010000:00010000:1.0:1773941597.909118:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d01488800 already unused, calling callback (ffffffffc12b4c20) 10000000:00000001:1.0:1773941597.909123:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000200:2.0:1773941597.909124:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291496320, portal 25 10000000:00010000:1.0:1773941597.909126:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a94 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941597.909127:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941597.909130:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631460992 : -113743078090624 : ffff988d203c1880) 00010000:00000001:1.0:1773941597.909138:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000100:00000040:2.0:1773941597.909139:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c1880 x1860112291496320/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941607 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941597.909141:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941597.909144:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941597.909147:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488800 refcount=5 00000020:00000001:1.0:1773941597.909151:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00000100:00000001:2.0:1773941597.909153:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:1.0:1773941597.909154:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00000100:00000040:2.0:1773941597.909159:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00000001:1.0:1773941597.909160:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000010:2.0:1773941597.909163:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516908. 00010000:00010000:1.0:1773941597.909164:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a94 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:2.0:1773941597.909168:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291496320, offset 0 10000000:00000001:1.0:1773941597.909171:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941597.909174:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a94 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941597.909175:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 10000000:01000000:1.0:1773941597.909182:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:2.0:1773941597.909183:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 10000000:00000040:1.0:1773941597.909189:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000400:00000200:2.0:1773941597.909192:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 10000000:00000001:1.0:1773941597.909193:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941597.909198:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00010000:1.0:1773941597.909200:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000400:00000200:2.0:1773941597.909203:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00000001:1.0:1773941597.909205:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:2.0:1773941597.909209:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941597.909212:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941597.909212:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941597.909217:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000400:00000200:2.0:1773941597.909218:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[154, 155] 00010000:00000001:1.0:1773941597.909222:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:2.0:1773941597.909224:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00010000:00000001:1.0:1773941597.909228:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941597.909229:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [266:266:256:4294967295] 192.168.204.114@tcp LPNI seq info [156:156:8:4294967295] 00000020:00000040:1.0:1773941597.909235:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d01488800 with handle 0x8e86767622e2a94 from hash 00000400:00000200:2.0:1773941597.909236:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941597.909242:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941597.909246:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941597.909250:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:2.0:1773941597.909253:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00010000:00000001:1.0:1773941597.909257:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:1.0:1773941597.909261:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941597.909264:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000800:00000200:2.0:1773941597.909265:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00010000:00000001:1.0:1773941597.909268:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941597.909271:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941597.909272:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941597.909274:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:2.0:1773941597.909275:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0dbd00. 00010000:00010000:1.0:1773941597.909277:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a94 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:2.0:1773941597.909286:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00010000:00000001:1.0:1773941597.909287:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941597.909290:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941597.909293:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941597.909294:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941597.909295:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941597.909296:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941597.909297:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941597.909299:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a94 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:1.0:1773941597.909306:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d04dd4e00 count: 0 00000800:00000200:2.0:1773941597.909307:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:2.0:1773941597.909314:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0dbd00 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:1.0:1773941597.909319:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941597.909321:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941597.909325:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941597.909331:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c0700 x1860112291496192/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941613 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941597.909351:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1773941597.909354:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941597.909355:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941597.909359:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d11188400. 00000400:00000200:2.0:1773941597.909363:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 02000000:00000001:1.0:1773941597.909364:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1773941597.909367:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294989505 00000020:00000001:1.0:1773941597.909368:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941597.909371:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941597.909375:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941597.909378:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294989503 02000000:00000010:1.0:1773941597.909378:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d181fc400. 02000000:00000001:1.0:1773941597.909383:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1773941597.909385:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294989505 02000000:00000001:1.0:1773941597.909386:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1773941597.909391:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294989503 00000100:00000010:1.0:1773941597.909395:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c0700. 00000100:00000001:2.0:1773941597.909397:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941597.909400:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941597.909401:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941597.909403:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941597.909404:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941597.909406:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941597.909408:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:01000000:1.0:1773941597.909410:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941597.909414:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000100:00000001:2.0:1773941597.909417:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 10000000:00000010:1.0:1773941597.909419:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d04d674c0. 00000100:00000001:2.0:1773941597.909420:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000800:00000200:0.0:1773941597.909422:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:2.0:1773941597.909426:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000010:1.0:1773941597.909427:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d181fc400. 00000800:00000010:0.0:1773941597.909428:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0dbd00. 00000100:00000001:2.0:1773941597.909429:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941597.909433:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941597.909435:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d181fc200. 00000100:00000001:2.0:1773941597.909437:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941597.909437:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000040:00000010:1.0:1773941597.909440:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d181fc800. 00000020:00000010:1.0:1773941597.909447:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d18b18000. 00000400:00000200:0.0:1773941597.909448:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000020:00000010:1.0:1773941597.909455:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d181fce00. 00000400:00000200:0.0:1773941597.909455:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516908 00000400:00000010:0.0:1773941597.909457:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516908. 80000000:00000010:1.0:1773941597.909461:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d054f3a20. 00000100:00000001:0.0:1773941597.909464:0:2393:0:(events.c:34:request_out_callback()) Process entered 00040000:00000010:1.0:1773941597.909467:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d181fd000. 00000100:00000200:0.0:1773941597.909471:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c1880 x1860112291496320/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941607 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 20000000:00000010:1.0:1773941597.909474:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d181fd800. 10000000:00000040:1.0:1773941597.909481:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 00000100:00000001:0.0:1773941597.909483:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 10000000:00000001:1.0:1773941597.909484:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000040:0.0:1773941597.909489:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c1880 x1860112291496320/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941607 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:1.0:1773941597.909490:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d181fc200. 00000040:00000010:1.0:1773941597.909497:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d181fc800. 00000100:00000001:0.0:1773941597.909502:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941597.909503:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d18b18000. 00000100:00000001:0.0:1773941597.909505:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:1.0:1773941597.909508:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d181fce00. 80000000:00000010:1.0:1773941597.909512:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d054f3a20. 00040000:00000010:1.0:1773941597.909516:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d181fd000. 20000000:00000010:1.0:1773941597.909519:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d181fd800. 00000020:00000010:1.0:1773941597.909523:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d181fc400. 10000000:00000010:1.0:1773941597.909525:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d04d674c0. 10000000:01000000:1.0:1773941597.909527:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941597.909530:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941597.909532:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941597.909534:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941597.909536:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941597.909537:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941597.909539:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941597.910949:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941597.910956:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941597.910959:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941597.910993:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941597.910998:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941597.911003:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941597.911010:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941597.911022:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941597.911027:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941597.911032:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324608180 00000400:00000200:0.0:1773941597.911036:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941597.911040:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0x1159 [1] + 0 00000800:00000001:0.0:1773941597.911046:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941597.911054:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941597.911056:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941597.911060:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941597.911065:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941597.911069:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c1880 x1860112291496320/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941607 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941597.911081:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c1880 x1860112291496320/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941607 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941597.911101:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941597.911107:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941597.911110:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941597.911128:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941597.911131:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941597.911133:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941597.911135:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941597.911136:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941597.911139:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941597.911141:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941597.911143:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941597.911147:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c1880 x1860112291496320/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941607 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941597.911154:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c1880 x1860112291496320/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941607 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941597.911159:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941597.911160:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941597.911161:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941597.911163:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941597.911164:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941597.911165:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941597.911171:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941597.911174:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=98 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941597.911177:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=0 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941597.911179:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941597.911180:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941597.911182:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941597.911184:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941597.911186:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941597.911189:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c1880 x1860112291496320/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941607 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941597.911194:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941597.911195:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941597.911197:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941597.911199:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941597.911204:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941597.911205:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941597.911206:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941597.911208:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941597.911209:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941597.911211:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941597.911213:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:2.0:1773941597.911216:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941597 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941597.911218:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941597.911223:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:2.0:1773941597.911224:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941597.911226:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941597.911228:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941597.911229:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941597.911232:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:2.0:1773941597.911233:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941597.911235:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941597.911238:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941597.911240:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:2.0:1773941597.911245:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941597.911478:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941597.911483:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941597.911489:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c1880 x1860112291496320/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941607 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941597.911501:0:14801:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:3.0:1773941597.911507:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291496320:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00080000:2.0:1773941597.911512:0:14801:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00000001:3.0:1773941597.911513:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941597.911517:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00100000:2.0:1773941597.911518:0:14801:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000040:3.0:1773941597.911522:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c1880 x1860112291496320/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941607 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941597.911523:0:14801:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941597.911526:0:14801:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:2.0:1773941597.911531:0:14801:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:3.0:1773941597.911534:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:2.0:1773941597.911536:0:14801:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:3.0:1773941597.911537:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941597.911542:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a5000. 00000001:00000001:2.0:1773941597.911542:0:14801:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:2.0:1773941597.911546:0:14801:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:3.0:1773941597.911547:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:2.0:1773941597.911548:0:14801:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000001:3.0:1773941597.911550:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:2.0:1773941597.911551:0:14801:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1773941597.911553:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 10000000:00000001:2.0:1773941597.911554:0:14801:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941597.911558:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941597.911558:0:14801:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000010:3.0:1773941597.911562:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d11189000. 00000100:00080000:2.0:1773941597.911564:0:14801:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 02000000:00000001:3.0:1773941597.911566:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941597.911569:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1773941597.911570:0:14801:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000010:3.0:1773941597.911572:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c1880. 00000100:00000001:2.0:1773941597.911573:0:14801:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:2.0:1773941597.911576:0:14801:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941597.911577:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00080000:2.0:1773941597.911579:0:14801:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:3.0:1773941597.911580:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941597.911583:0:14801:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:3.0:1773941597.911584:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:01000000:2.0:1773941597.911586:0:14801:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000100:00000001:3.0:1773941597.911589:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 10000000:00000040:2.0:1773941597.911589:0:14801:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00000001:3.0:1773941597.911592:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941597.911594:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941597.911595:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1773941597.911602:0:14801:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:2.0:1773941597.911604:0:14801:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941597.911606:0:14801:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:2.0:1773941597.911610:0:14801:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:2.0:1773941597.911613:0:14801:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941597.911615:0:14801:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941597.911616:0:14801:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941597.911619:0:14801:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941597.911621:0:14801:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941597.914994:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d04dd4e00. 00010000:00000010:1.1:1773941597.915006:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d01488800. 00000020:00000001:2.0:1773941598.203173:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941598.203180:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941598.203186:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941599.227183:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294989505, at 4294989568 00000100:00000040:0.0:1773941599.227207:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294989505, at 4294989568 00000100:00000010:1.0:1773941599.227235:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3420. 00000100:00000010:0.0:1773941599.227242:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f33c0. 00000020:00000001:2.0:1773941599.227335:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941599.227371:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941599.227378:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941599.291148:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941599.291158:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941599.291164:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941599.291167:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941599.291169:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941599.291171:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941599.291174:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941599.291177:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941599.291181:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:1.0:1773941599.291184:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941599.291187:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941599.291201:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c0700. 00000020:00000040:1.0:1773941599.291207:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941599.291213:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941599.291217:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941599.291224:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941599.291234:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d181fd000. 00000100:00000001:1.0:1773941599.291243:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941599.291249:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941599.291252:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941599.291262:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d04dd4e00. 00010000:00000001:1.0:1773941599.291269:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941599.291274:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d01488800. 00000020:00000001:1.0:1773941599.291279:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941599.291282:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d01488800 with handle 0x8e86767622e2a9b to hash 00000020:00000001:1.0:1773941599.291285:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941599.291287:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941599.291291:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941599.291296:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941599.291300:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941599.291305:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a9b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941599.291352:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a9b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941599.291388:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a9b lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941599.291396:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941599.291398:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941599.291406:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941599.291409:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330528610752 : -113743180940864 : ffff988d1a1ab9c0) 00000100:00000001:1.0:1773941599.291413:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941599.291415:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00000100:00000001:1.0:1773941599.291421:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941599.291426:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941599.291428:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941599.291431:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941599.291444:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c0700 x1860112291496448/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941599.291456:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941599.291458:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941599.291463:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941599.291466:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941599.291470:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941599.291474:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291496448:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941599.291479:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941599.291486:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941599.291488:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941599.291492:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941599.291496:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d11188400. 02000000:00000001:1.0:1773941599.291499:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941599.291508:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941599.291513:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c5d8. 00000400:00000010:1.0:1773941599.291521:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000100:00000200:1.0:1773941599.291529:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291496448, portal 25 00000100:00000001:1.0:1773941599.291531:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941599.291541:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00000100:00000040:1.0:1773941599.291549:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c0700 x1860112291496448/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941615 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941599.291559:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941599.291562:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941599.291567:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113ded48. 00000100:00000200:1.0:1773941599.291571:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291496448, offset 0 00000400:00000200:1.0:1773941599.291578:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941599.291589:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941599.291598:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941599.291605:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941599.291611:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941599.291616:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941599.291619:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941599.291624:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[156, 155] 00000400:00000200:1.0:1773941599.291629:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:1.0:1773941599.291634:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [267:267:256:4294967295] 192.168.204.124@tcp LPNI seq info [157:157:8:4294967295] 00000400:00000200:1.0:1773941599.291639:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941599.291652:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:1.0:1773941599.291660:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:1.0:1773941599.291666:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfd300. 00000800:00000200:1.0:1773941599.291674:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:1.0:1773941599.291683:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:1.0:1773941599.291689:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfd300 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941599.291716:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941599.291721:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941599.291724:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294990887 00000100:00000010:1.0:1773941599.291730:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3420. 00000100:00000001:1.0:1773941599.291738:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1773941599.291748:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294990887 00000100:00000010:1.0:1773941599.291760:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3a20. 00000100:00000001:1.0:1773941599.291769:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1773941599.291774:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941599.291777:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941599.291781:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941599.291783:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941599.291787:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 11 seconds 00000100:00000001:1.0:1773941599.291792:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941599.291808:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941599.292028:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941599.292045:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfd300. 00000400:00000200:0.0:1773941599.292057:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941599.292073:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941599.292086:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113ded48 00000400:00000010:0.0:1773941599.292091:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113ded48. 00000100:00000001:0.0:1773941599.292100:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941599.292114:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c0700 x1860112291496448/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941615 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941599.292136:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941599.292144:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d203c0700 x1860112291496448/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941615 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941599.292193:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941599.292198:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941599.293459:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941599.293475:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941599.293479:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941599.293482:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941599.293491:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941599.293499:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941599.293508:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941599.293514:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941599.293522:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941599.293527:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324608200 00000400:00000200:0.0:1773941599.293533:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941599.293541:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0x1161 [1] + 224 00000800:00000001:0.0:1773941599.293553:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941599.293563:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941599.293566:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941599.293571:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941599.293577:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941599.293584:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c0700 x1860112291496448/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941615 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941599.293597:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d203c0700 x1860112291496448/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941615 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941599.293622:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941599.293632:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941599.293635:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941599.293738:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941599.293749:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941599.293752:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941599.293756:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:1.0:1773941599.293765:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:1.0:1773941599.293768:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:1.0:1773941599.293772:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941599.293782:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c0700 x1860112291496448/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941615 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941599.293797:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c0700 x1860112291496448/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941615 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941599.293806:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941599.293808:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941599.293812:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941599.293815:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941599.293818:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941599.293821:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941599.293824:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941599.293831:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=100 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941599.293836:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941599.293839:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941599.293844:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:1.0:1773941599.293848:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941599.293852:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941599.293856:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941599.293873:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941599.293879:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941599.293884:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941599.293887:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941599.293890:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941599.293892:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941599.293897:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d203c0700 x1860112291496448/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941615 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941599.293906:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941599.293908:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941599.293910:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941599.293914:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941599.293917:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941599.293919:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941599.293921:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941599.293922:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941599.293924:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941599.293925:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941599.293927:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941599.293930:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941599.293933:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941599.293936:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941599.293940:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 320 00000400:00000200:1.0:1773941599.293947:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941599.293953:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941599.293958:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941599.293991:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941599.293995:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941599.293997:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941599.294002:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941599.294004:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941599.294007:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941599.294016:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941599.294019:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941599.294022:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941599.294026:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941599.294028:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941599.294031:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941599.294033:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941599.294037:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941599.294041:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941599.294044:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941599.294047:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941599.294051:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941599.294054:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941599.294057:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941599.294059:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941599.294063:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c2680. 00000020:00000040:1.0:1773941599.294067:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941599.294070:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941599.294072:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941599.294076:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941599.294078:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941599.294081:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941599.294086:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d1118a000. 00000100:00000001:1.0:1773941599.294091:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941599.294098:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c2680 x1860112291496576/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941599.294108:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941599.294111:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941599.294117:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000097d1012f] to pc [ptlrpcd_rcv+-1] req@ffff988d203c2680 x1860112291496576/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941599.294146:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941599.294150:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941599.294158:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941599.294161:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941599.294164:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941599.294166:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941599.294169:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941599.294172:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941599.294185:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941599.294192:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c0700 x1860112291496448/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941615 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941599.294202:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941599.294207:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941599.294211:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941599.294214:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941599.294219:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c0700 x1860112291496448/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941615 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941599.294228:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291496448:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941599.294234:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941599.294237:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941599.294238:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941599.294249:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941599.294253:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1773941599.294256:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000040:1.0:1773941599.294260:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c0700 x1860112291496448/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941615 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941599.294265:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941599.294280:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941599.294285:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941599.294294:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000040:3.0:1773941599.294296:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c2680 x1860112291496576/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941599.294296:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:1.0:1773941599.294310:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000001:3.0:1773941599.294350:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:1.0:1773941599.294351:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941599.294360:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:3.0:1773941599.294361:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941599.294365:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488800 refcount=4 02000000:00000001:3.0:1773941599.294371:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:1.0:1773941599.294372:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 02000000:00000001:3.0:1773941599.294379:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941599.294399:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941599.294407:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00010000:1.0:1773941599.294414:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a9b lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:3.0:1773941599.294427:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291496576:192.168.204.114@tcp:250:ll_cfg_requeue.0 00010000:00000001:1.0:1773941599.294451:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941599.294452:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941599.294463:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00010000:1.0:1773941599.294466:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a9b lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941599.294470:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941599.294487:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:1.0:1773941599.294496:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 02000000:00000010:3.0:1773941599.294497:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03523000. 02000000:00000001:3.0:1773941599.294504:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941599.294504:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a9b lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941599.294532:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941599.294547:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00010000:00000001:1.0:1773941599.294547:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941599.294560:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:1.0:1773941599.294566:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a9b lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:3.0:1773941599.294569:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00010000:00000001:1.0:1773941599.294595:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941599.294602:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941599.294607:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000100:00000200:3.0:1773941599.294609:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291496576, portal 25 00010000:00010000:1.0:1773941599.294613:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a9b lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941599.294618:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941599.294630:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631464576 : -113743078087040 : ffff988d203c2680) 00010000:00010000:1.0:1773941599.294633:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d01488800 already unused, calling callback (ffffffffc12b4c20) 10000000:00000001:1.0:1773941599.294645:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941599.294651:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a9b lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:3.0:1773941599.294663:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c2680 x1860112291496576/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941609 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:1.0:1773941599.294666:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:1.0:1773941599.294671:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941599.294676:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941599.294681:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488800 refcount=5 00000100:00000001:3.0:1773941599.294698:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:1.0:1773941599.294698:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00010000:00000001:1.0:1773941599.294708:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112182272 : -113743597369344 : ffff988d01488800) 00000100:00000040:3.0:1773941599.294713:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00000001:1.0:1773941599.294718:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000010:3.0:1773941599.294721:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acd48. 00010000:00010000:1.0:1773941599.294724:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a9b lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:3.0:1773941599.294730:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291496576, offset 0 10000000:00000001:1.0:1773941599.294751:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:3.0:1773941599.294763:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 10000000:00010000:1.0:1773941599.294764:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a9b lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:1.0:1773941599.294787:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:3.0:1773941599.294802:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 10000000:00000040:1.0:1773941599.294803:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941599.294811:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941599.294818:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000400:00000200:3.0:1773941599.294819:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:1.0:1773941599.294826:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:3.0:1773941599.294833:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:1.0:1773941599.294833:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941599.294851:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000400:00000200:3.0:1773941599.294860:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00000001:1.0:1773941599.294862:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941599.294872:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941599.294878:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000020:00000040:1.0:1773941599.294882:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d01488800 with handle 0x8e86767622e2a9b from hash 00000400:00000200:3.0:1773941599.294890:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941599.294893:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941599.294902:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:3.0:1773941599.294903:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[156, 157] 00010000:00000001:1.0:1773941599.294912:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941599.294919:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000400:00000200:3.0:1773941599.294921:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00010000:00000001:1.0:1773941599.294924:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941599.294928:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000400:00000200:3.0:1773941599.294935:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [268:268:256:4294967295] 192.168.204.114@tcp LPNI seq info [158:158:8:4294967295] 00010000:00000001:1.0:1773941599.294938:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941599.294949:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:3.0:1773941599.294955:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941599.294958:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941599.294965:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941599.294971:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a9b lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:1.0:1773941599.295037:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941599.295049:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941599.295052:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:1.0:1773941599.295056:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000400:00000200:3.0:1773941599.295067:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00010000:00000001:1.0:1773941599.295067:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941599.295080:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941599.295084:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941599.295089:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d01488800/0x8e86767622e2a9b lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:1.0:1773941599.295122:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d04dd4e00 count: 0 00000800:00000200:3.0:1773941599.295123:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:3.0:1773941599.295142:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74f00. 00010000:00000001:1.0:1773941599.295152:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941599.295159:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000800:00000200:3.0:1773941599.295164:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000100:00000001:1.0:1773941599.295166:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941599.295181:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c0700 x1860112291496448/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941615 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1773941599.295202:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000100:00000001:1.0:1773941599.295205:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941599.295210:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:1.0:1773941599.295222:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d11188400. 00000800:00000200:3.0:1773941599.295224:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74f00 type 1, nob 616 niov 1 nkiov 1 02000000:00000001:1.0:1773941599.295228:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941599.295234:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941599.295242:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941599.295247:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:1.0:1773941599.295251:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d181fd000. 02000000:00000001:1.0:1773941599.295255:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941599.295257:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941599.295259:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c0700. 00000100:00000001:1.0:1773941599.295265:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941599.295267:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941599.295269:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941599.295280:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941599.295286:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:1.0:1773941599.295292:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000400:00000200:3.0:1773941599.295293:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 10000000:00000010:1.0:1773941599.295302:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d04d67c40. 00000100:00000001:3.0:1773941599.295303:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294990891 00000020:00000010:1.0:1773941599.295340:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d181fd000. 00000100:00000001:3.0:1773941599.295344:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294990887 00000020:00000010:1.0:1773941599.295351:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d181fce00. 00000100:00000001:3.0:1773941599.295356:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294990891 00000040:00000010:1.0:1773941599.295359:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d181fc800. 00000100:00000001:3.0:1773941599.295368:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294990887 00000020:00000010:1.0:1773941599.295371:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d18b18000. 00000100:00000001:3.0:1773941599.295378:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941599.295386:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941599.295394:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941599.295396:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d181fc200. 00000100:00000001:3.0:1773941599.295405:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 80000000:00000010:1.0:1773941599.295408:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d054f3d80. 00000800:00000200:0.0:1773941599.295410:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00040000:00000010:1.0:1773941599.295420:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d181fd600. 00000100:00000001:3.0:1773941599.295421:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000800:00000010:0.0:1773941599.295425:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74f00. 00000100:00000001:3.0:1773941599.295434:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 20000000:00000010:1.0:1773941599.295439:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d181fd800. 00000400:00000200:0.0:1773941599.295442:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1773941599.295444:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941599.295451:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 10000000:00000040:1.0:1773941599.295452:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 00000100:00000001:3.0:1773941599.295459:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941599.295462:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941599.295470:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941599.295473:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000020:00000010:1.0:1773941599.295484:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d181fce00. 00000400:00000200:0.0:1773941599.295492:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acd48 00000040:00000010:1.0:1773941599.295494:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d181fc800. 00000020:00000010:1.0:1773941599.295501:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941599.295507:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d181fc200. 80000000:00000010:1.0:1773941599.295521:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d054f3d80. 00000400:00000010:0.0:1773941599.295524:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acd48. 00040000:00000010:1.0:1773941599.295530:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d181fd600. 20000000:00000010:1.0:1773941599.295545:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d181fd800. 00000100:00000001:0.0:1773941599.295547:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000020:00000010:1.0:1773941599.295552:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d181fd000. 10000000:00000010:1.0:1773941599.295559:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d04d67c40. 10000000:01000000:1.0:1773941599.295571:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 00000100:00000200:0.0:1773941599.295573:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c2680 x1860112291496576/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941609 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000001:1.0:1773941599.295591:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941599.295601:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941599.295607:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941599.295614:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941599.295618:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941599.295628:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000100:00000001:0.0:1773941599.295646:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941599.295654:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c2680 x1860112291496576/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941609 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941599.295665:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941599.295669:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941599.298135:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941599.298149:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941599.298154:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941599.298158:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941599.298166:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941599.298175:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941599.298187:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941599.298192:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941599.298198:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941599.298204:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324608280 00000400:00000200:0.0:1773941599.298210:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941599.298219:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0x1171 [1] + 0 00000800:00000001:0.0:1773941599.298228:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941599.298256:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941599.298262:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941599.298269:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941599.298278:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941599.298287:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c2680 x1860112291496576/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941609 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941599.298305:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c2680 x1860112291496576/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941609 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941599.298358:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941599.298388:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941599.298395:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941599.298401:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941599.298406:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941599.298413:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941599.298415:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941599.298419:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941599.298427:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941599.298430:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941599.298433:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941599.298441:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c2680 x1860112291496576/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941609 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941599.298460:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c2680 x1860112291496576/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941609 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941599.298496:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941599.298500:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941599.298505:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941599.298509:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941599.298513:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941599.298518:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941599.298522:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941599.298533:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=100 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941599.298540:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941599.298544:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941599.298547:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941599.298552:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941599.298556:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941599.298561:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941599.298571:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c2680 x1860112291496576/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941609 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941599.298584:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941599.298588:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941599.298592:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941599.298596:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941599.298616:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941599.298622:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941599.298626:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941599.298630:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941599.298635:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941599.298640:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941599.298645:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941599.298652:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941599 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941599.298657:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941599.298669:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941599.298673:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941599.298679:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941599.298682:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941599.298693:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941599.298699:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941599.298702:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941599.298705:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941599.298711:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941599.298715:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941599.298718:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941599.299140:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941599.299149:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941599.299164:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c2680 x1860112291496576/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941609 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941599.299174:0:14803:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:2.0:1773941599.299192:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c2680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291496576:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00080000:3.0:1773941599.299195:0:14803:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:3.0:1773941599.299211:0:14803:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941599.299211:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941599.299218:0:14803:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:2.0:1773941599.299221:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 10000000:01000000:3.0:1773941599.299224:0:14803:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:3.0:1773941599.299232:0:14803:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000040:2.0:1773941599.299235:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c2680 x1860112291496576/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941609 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000001:00000001:3.0:1773941599.299240:0:14803:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:3.0:1773941599.299251:0:14803:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:3.0:1773941599.299261:0:14803:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:2.0:1773941599.299262:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:3.0:1773941599.299265:0:14803:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:2.0:1773941599.299270:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:3.0:1773941599.299271:0:14803:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:3.0:1773941599.299278:0:14803:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941599.299281:0:14803:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000010:2.0:1773941599.299282:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03523000. 00000100:00080000:3.0:1773941599.299287:0:14803:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 02000000:00000001:2.0:1773941599.299294:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1773941599.299295:0:14803:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:3.0:1773941599.299303:0:14803:0:(recover.c:147:ptlrpc_resend()) Process entered 00000020:00000001:2.0:1773941599.299304:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:3.0:1773941599.299307:0:14803:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941599.299331:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00080000:3.0:1773941599.299333:0:14803:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:3.0:1773941599.299340:0:14803:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941599.299343:0:14803:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000020:00000001:2.0:1773941599.299345:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 10000000:00000040:3.0:1773941599.299346:0:14803:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 02000000:00000010:2.0:1773941599.299356:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d1118a000. 02000000:00000001:2.0:1773941599.299366:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1773941599.299370:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:3.0:1773941599.299373:0:14803:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 00000100:00000010:2.0:1773941599.299374:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c2680. 10000000:00000001:3.0:1773941599.299377:0:14803:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941599.299382:0:14803:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941599.299388:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:02000000:3.0:1773941599.299390:0:14803:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:2.0:1773941599.299396:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941599.299398:0:14803:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941599.299409:0:14803:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:2.0:1773941599.299409:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941599.299421:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000020:00000040:3.0:1773941599.299425:0:14803:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941599.299428:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941599.299430:0:14803:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941599.299435:0:14803:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941599.299438:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941599.299443:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941599.301048:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d04dd4e00. 00010000:00000010:1.1:1773941599.301060:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d01488800. 00000020:00000001:1.0:1773941600.251234:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1773941600.251243:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941600.251247:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941600.571041:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294990891, at 4294990912 00000100:00000040:0.0:1773941600.571041:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294990891, at 4294990912 00000100:00000010:1.0:1773941600.571059:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3a20. 00000100:00000010:0.0:1773941600.571061:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3420. 10000000:00000001:0.0:1773941600.955126:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1773941600.955136:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941600.955142:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941600.955145:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941600.955147:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941600.955148:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1773941600.955152:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:0.0:1773941600.955154:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:0.0:1773941600.955157:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:0.0:1773941600.955160:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1773941600.955162:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:0.0:1773941600.955176:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10837b80. 00000020:00000040:0.0:1773941600.955182:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941600.955187:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941600.955191:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941600.955198:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941600.955208:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065c5c00. 00000100:00000001:0.0:1773941600.955216:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941600.955221:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1773941600.955224:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1773941600.955235:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d21ec1800. 00010000:00000001:0.0:1773941600.955242:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1773941600.955245:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d00030c00. 00000020:00000001:0.0:1773941600.955250:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:0.0:1773941600.955253:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d00030c00 with handle 0x8e86767622e2aa2 to hash 00000020:00000001:0.0:1773941600.955255:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:0.0:1773941600.955256:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330090851328 : -113743618700288 : ffff988d00030c00) 00010000:00000001:0.0:1773941600.955259:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330090851328 : -113743618700288 : ffff988d00030c00) 00010000:00000001:0.0:1773941600.955263:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941600.955267:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941600.955271:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e2aa2 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941600.955355:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e2aa2 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941600.955370:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e2aa2 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941600.955377:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1773941600.955380:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941600.955388:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb66c0. 00000100:00000001:0.0:1773941600.955390:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330139633344 : -113743569918272 : ffff988d02eb66c0) 00000100:00000001:0.0:1773941600.955394:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941600.955396:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367703936 : -113743341847680 : ffff988d10837b80) 00000100:00000001:0.0:1773941600.955402:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941600.955407:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941600.955410:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941600.955413:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941600.955426:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10837b80 x1860112291496704/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941600.955436:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941600.955439:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941600.955443:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941600.955446:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941600.955449:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941600.955453:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10837b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291496704:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941600.955457:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941600.955465:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941600.955468:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941600.955471:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941600.955476:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d02ce7400. 02000000:00000001:0.0:1773941600.955478:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941600.955488:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941600.955499:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0014b160. 00000400:00000010:0.0:1773941600.955508:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833990. 00000100:00000200:0.0:1773941600.955517:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291496704, portal 25 00000100:00000001:0.0:1773941600.955519:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941600.955520:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367703936 : -113743341847680 : ffff988d10837b80) 00000100:00000040:0.0:1773941600.955526:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10837b80 x1860112291496704/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941616 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941600.955535:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941600.955538:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941600.955541:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833198. 00000100:00000200:0.0:1773941600.955560:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291496704, offset 0 00000400:00000200:0.0:1773941600.955567:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941600.955577:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941600.955586:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941600.955592:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941600.955597:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941600.955601:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941600.955604:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941600.955607:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[158, 157] 00000400:00000200:0.0:1773941600.955610:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:0.0:1773941600.955615:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [269:269:256:4294967295] 192.168.204.124@tcp LPNI seq info [159:159:8:4294967295] 00000400:00000200:0.0:1773941600.955618:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941600.955627:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:0.0:1773941600.955650:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:0.0:1773941600.955656:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d113e9900. 00000800:00000200:0.0:1773941600.955665:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:0.0:1773941600.955675:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:0.0:1773941600.955682:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d113e9900 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1773941600.955708:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941600.955714:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941600.955718:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294992551 00000100:00000010:0.0:1773941600.955730:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580b40. 00000100:00000001:0.0:1773941600.955751:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1773941600.955759:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294992551 00000100:00000010:0.0:1773941600.955762:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580ea0. 00000100:00000001:0.0:1773941600.955766:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1773941600.955770:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941600.955773:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941600.955777:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941600.955779:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1773941600.955783:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb66c0 going to sleep for 11 seconds 00000100:00000001:0.0:1773941600.955787:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941600.955792:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941600.955927:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941600.955934:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d113e9900. 00000400:00000200:1.0:1773941600.955942:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941600.955949:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941600.955955:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833198 00000400:00000010:1.0:1773941600.955957:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833198. 00000100:00000001:1.0:1773941600.955991:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941600.955997:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10837b80 x1860112291496704/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941616 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941600.956008:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941600.956012:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d10837b80 x1860112291496704/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941616 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941600.956018:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941600.956020:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:0.0:1773941600.957822:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941600.957835:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941600.957838:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941600.957840:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941600.957846:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941600.957852:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:0.0:1773941600.957859:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941600.957863:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941600.957868:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941600.957872:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324608300 00000400:00000200:0.0:1773941600.957878:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941600.957896:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0x1179 [1] + 224 00000800:00000001:0.0:1773941600.957907:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941600.957919:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941600.957922:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941600.957927:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941600.957933:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941600.957940:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10837b80 x1860112291496704/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941616 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941600.957956:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d10837b80 x1860112291496704/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941616 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941600.958021:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941600.958030:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941600.958045:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941600.958059:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:1.0:1773941600.958068:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1773941600.958070:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:1.0:1773941600.958073:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0014b160. 00000400:00000200:1.0:1773941600.958077:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833990 00000400:00000010:1.0:1773941600.958079:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833990. 00000100:00000001:1.0:1773941600.958081:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941600.958087:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10837b80 x1860112291496704/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941616 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941600.958098:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10837b80 x1860112291496704/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941616 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941600.958105:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1773941600.958106:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941600.958109:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:1.0:1773941600.958112:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:1.0:1773941600.958114:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:1.0:1773941600.958116:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941600.958118:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:1.0:1773941600.958125:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=101 v=5 (5 0 0 0) 00000100:00001000:1.0:1773941600.958137:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:1.0:1773941600.958139:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:1.0:1773941600.958143:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:1.0:1773941600.958146:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941600.958149:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:1.0:1773941600.958152:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:1.0:1773941600.958169:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:1.0:1773941600.958174:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:1.0:1773941600.958177:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:1.0:1773941600.958180:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:1.0:1773941600.958181:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:1.0:1773941600.958184:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:1.0:1773941600.958188:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d10837b80 x1860112291496704/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941616 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941600.958194:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:1.0:1773941600.958195:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:1.0:1773941600.958197:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941600.958199:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:1.0:1773941600.958202:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:1.0:1773941600.958203:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941600.958205:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941600.958206:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941600.958207:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:1.0:1773941600.958208:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941600.958209:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:1.0:1773941600.958222:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1773941600.958224:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1773941600.958227:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:1.0:1773941600.958229:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 322 00000400:00000200:1.0:1773941600.958234:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941600.958238:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:1.0:1773941600.958243:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941600.958246:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941600.958248:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941600.958250:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:1.0:1773941600.958254:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:1.0:1773941600.958256:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:1.0:1773941600.958258:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941600.958265:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:1.0:1773941600.958267:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:1.0:1773941600.958270:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:1.0:1773941600.958273:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941600.958274:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:1.0:1773941600.958276:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:1.0:1773941600.958277:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:1.0:1773941600.958299:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:1.0:1773941600.958304:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:1.0:1773941600.958307:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941600.958311:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1773941600.958315:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941600.958319:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941600.958338:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:1.0:1773941600.958342:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941600.958349:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c0700. 00000020:00000040:1.0:1773941600.958353:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:1.0:1773941600.958358:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1773941600.958360:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941600.958364:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941600.958367:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941600.958370:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941600.958377:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d11188800. 00000100:00000001:1.0:1773941600.958384:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941600.958393:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d203c0700 x1860112291496832/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941600.958407:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941600.958411:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941600.958419:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000859dd066] to pc [ptlrpcd_rcv+-1] req@ffff988d203c0700 x1860112291496832/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941600.958445:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941600.958448:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:1.0:1773941600.958457:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941600.958460:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941600.958462:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941600.958464:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1773941600.958466:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb66c0 going to sleep for 1 seconds 00000100:00000001:1.0:1773941600.958468:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941600.958471:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941600.958477:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10837b80 x1860112291496704/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941616 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941600.958485:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941600.958488:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941600.958491:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1773941600.958493:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:1.0:1773941600.958498:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10837b80 x1860112291496704/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941616 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941600.958505:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1773941600.958507:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10837b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291496704:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:2.0:1773941600.958515:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941600.958516:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941600.958522:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:2.0:1773941600.958523:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941600.958525:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:1.0:1773941600.958529:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941600.958537:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10837b80 x1860112291496704/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941616 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941600.958541:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c0700 x1860112291496832/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941600.958570:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941600.958585:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb66c0. 00000100:00000001:1.0:1773941600.958588:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:1.0:1773941600.958589:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:1.0:1773941600.958591:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:1.0:1773941600.958596:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941600.958599:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941600.958602:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030c00 refcount=4 00000020:00000001:1.0:1773941600.958608:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090851328 : -113743618700288 : ffff988d00030c00) 00000100:00000001:2.0:1773941600.958614:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:1.0:1773941600.958619:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090851328 : -113743618700288 : ffff988d00030c00) 00000100:00000001:2.0:1773941600.958620:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941600.958626:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e2aa2 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941600.958628:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941600.958649:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00010000:00000001:1.0:1773941600.958666:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 02000000:00000001:2.0:1773941600.958669:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941600.958673:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e2aa2 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:2.0:1773941600.958683:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291496832:192.168.204.114@tcp:250:ll_cfg_requeue.0 00010000:00000001:1.0:1773941600.958686:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:1.0:1773941600.958692:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e2aa2 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941600.958699:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941600.958709:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00000001:1.0:1773941600.958710:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941600.958717:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 02000000:00000001:2.0:1773941600.958719:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941600.958722:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e2aa2 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941600.958728:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:1.0:1773941600.958736:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941600.958739:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941600.958741:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a7400. 00010000:00000001:1.0:1773941600.958743:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:1.0:1773941600.958748:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e2aa2 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:2.0:1773941600.958750:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941600.958765:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d00030c00 already unused, calling callback (ffffffffc12b4c20) 00000400:00000200:2.0:1773941600.958767:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 10000000:00000001:1.0:1773941600.958770:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941600.958776:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e2aa2 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:2.0:1773941600.958778:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00010000:00000001:1.0:1773941600.958791:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000400:00000010:2.0:1773941600.958795:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00010000:00000001:1.0:1773941600.958795:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:1.0:1773941600.958799:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:1.0:1773941600.958804:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030c00 refcount=5 00000100:00000200:2.0:1773941600.958808:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291496832, portal 25 00000020:00000001:1.0:1773941600.958818:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090851328 : -113743618700288 : ffff988d00030c00) 00000100:00000001:2.0:1773941600.958819:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00000001:1.0:1773941600.958825:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090851328 : -113743618700288 : ffff988d00030c00) 00000100:00000001:2.0:1773941600.958827:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631456512 : -113743078095104 : ffff988d203c0700) 00010000:00000001:1.0:1773941600.958849:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:1.0:1773941600.958853:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e2aa2 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:2.0:1773941600.958858:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c0700 x1860112291496832/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941610 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000001:1.0:1773941600.958862:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:1.0:1773941600.958867:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e2aa2 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:2.0:1773941600.958887:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 10000000:01000000:1.0:1773941600.958887:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:1.0:1773941600.958893:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:1.0:1773941600.958896:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941600.958897:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00010000:1.0:1773941600.958900:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:1.0:1773941600.958903:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000010:2.0:1773941600.958905:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516908. 00010000:00000001:1.0:1773941600.958907:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:1.0:1773941600.958911:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000100:00000200:2.0:1773941600.958913:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291496832, offset 0 00010000:00000001:1.0:1773941600.958913:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941600.958917:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:1.0:1773941600.958921:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d00030c00 with handle 0x8e86767622e2aa2 from hash 00000400:00000200:2.0:1773941600.958926:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00000001:1.0:1773941600.958927:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:1.0:1773941600.958948:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941600.958950:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941600.958952:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000400:00000200:2.0:1773941600.958954:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:1.0:1773941600.958955:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:1.0:1773941600.958960:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:1.0:1773941600.958964:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:2.0:1773941600.958965:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:1.0:1773941600.958967:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941600.958970:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941600.958973:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:2.0:1773941600.958975:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00010000:1.0:1773941600.958977:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e2aa2 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:2.0:1773941600.958982:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00000001:1.0:1773941600.958988:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:2.0:1773941600.958991:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941600.958995:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941600.958999:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000400:00000200:2.0:1773941600.959000:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:1.0:1773941600.959001:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:1.0:1773941600.959003:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:1.0:1773941600.959004:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941600.959005:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:1.0:1773941600.959023:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d00030c00/0x8e86767622e2aa2 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:1.0:1773941600.959064:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d21ec1800 count: 0 00010000:00000001:1.0:1773941600.959097:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:1.0:1773941600.959100:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941600.959104:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941600.959112:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10837b80 x1860112291496704/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941616 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941600.959121:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1773941600.959126:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000400:00000200:2.0:1773941600.959127:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[158, 159] 02000000:00000010:1.0:1773941600.959132:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d02ce7400. 00000400:00000200:2.0:1773941600.959156:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 02000000:00000001:1.0:1773941600.959158:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1773941600.959166:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000400:00000200:2.0:1773941600.959167:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [270:270:256:4294967295] 192.168.204.114@tcp LPNI seq info [160:160:8:4294967295] 00000020:00000040:1.0:1773941600.959172:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941600.959181:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 00000400:00000200:2.0:1773941600.959184:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 02000000:00000010:1.0:1773941600.959187:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065c5c00. 02000000:00000001:1.0:1773941600.959196:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1773941600.959201:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:1.0:1773941600.959207:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10837b80. 00000400:00000200:2.0:1773941600.959208:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000100:00000001:1.0:1773941600.959215:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941600.959222:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000800:00000200:2.0:1773941600.959223:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 10000000:00000001:1.0:1773941600.959227:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:1.0:1773941600.959233:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000800:00000010:2.0:1773941600.959234:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0dbd00. 10000000:00000001:1.0:1773941600.959237:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:1.0:1773941600.959245:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d04d67580. 00000800:00000200:2.0:1773941600.959246:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000020:00000010:1.0:1773941600.959257:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d181fd800. 00000800:00000200:2.0:1773941600.959267:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000020:00000010:1.0:1773941600.959270:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d181fd600. 00000800:00000200:2.0:1773941600.959273:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0dbd00 type 1, nob 616 niov 1 nkiov 1 00000040:00000010:1.0:1773941600.959320:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d181fc200. 00000100:00000001:2.0:1773941600.959330:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:1.0:1773941600.959330:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d18b18000. 00000400:00000200:2.0:1773941600.959335:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941600.959338:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294992555 00000100:00000001:2.0:1773941600.959344:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294992551 00000100:00000001:2.0:1773941600.959348:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294992555 00000100:00000001:2.0:1773941600.959351:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294992551 00000100:00000001:2.0:1773941600.959356:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941600.959359:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941600.959361:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941600.959381:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941600.959389:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941600.959391:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1773941600.959393:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941600.959395:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941600.959397:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941600.959400:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941600.959438:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941600.959443:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0dbd00. 00000400:00000200:0.0:1773941600.959449:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941600.959458:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941600.959463:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516908 00000400:00000010:0.0:1773941600.959465:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516908. 00000100:00000001:0.0:1773941600.959469:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941600.959475:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c0700 x1860112291496832/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941610 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941600.959486:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941600.959492:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c0700 x1860112291496832/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941610 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941600.959501:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941600.959505:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:1.0:1773941600.959506:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d181fc800. 80000000:00000010:1.0:1773941600.959513:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d054f3a20. 00040000:00000010:1.0:1773941600.959517:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d181fce00. 20000000:00000010:1.0:1773941600.959521:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d181fc400. 10000000:00000040:1.0:1773941600.959528:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:1.0:1773941600.959530:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:1.0:1773941600.959536:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d181fd600. 00000040:00000010:1.0:1773941600.959542:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d181fc200. 00000020:00000010:1.0:1773941600.959556:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d18b18000. 00000020:00000010:1.0:1773941600.959560:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d181fc800. 80000000:00000010:1.0:1773941600.959562:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d054f3a20. 00040000:00000010:1.0:1773941600.959565:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d181fce00. 20000000:00000010:1.0:1773941600.959567:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d181fc400. 00000020:00000010:1.0:1773941600.959569:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d181fd800. 10000000:00000010:1.0:1773941600.959571:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d04d67580. 10000000:01000000:1.0:1773941600.959573:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:1.0:1773941600.959576:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:1.0:1773941600.959579:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:1.0:1773941600.959581:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:1.0:1773941600.959583:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:1.0:1773941600.959584:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941600.959586:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:0.0:1773941600.961047:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941600.961055:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:0.0:1773941600.961058:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941600.961061:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941600.961065:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941600.961086:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:0.0:1773941600.961092:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941600.961094:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941600.961097:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941600.961100:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324608380 00000400:00000200:0.0:1773941600.961104:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941600.961107:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0x1189 [1] + 0 00000800:00000001:0.0:1773941600.961111:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941600.961118:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:0.0:1773941600.961120:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941600.961132:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:0.0:1773941600.961135:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941600.961139:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c0700 x1860112291496832/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941610 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941600.961148:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d203c0700 x1860112291496832/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941610 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941600.961185:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:0.0:1773941600.961193:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:0.0:1773941600.961197:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941600.961255:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941600.961261:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:2.0:1773941600.961267:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941600.961269:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941600.961272:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941600.961311:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000400:00000010:2.0:1773941600.961339:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941600.961342:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:2.0:1773941600.961348:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c0700 x1860112291496832/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941610 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:2.0:1773941600.961357:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c0700 x1860112291496832/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941610 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941600.961364:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941600.961366:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941600.961368:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:2.0:1773941600.961369:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941600.961371:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:2.0:1773941600.961372:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941600.961375:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:2.0:1773941600.961378:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=101 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941600.961381:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:2.0:1773941600.961383:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941600.961384:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941600.961387:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941600.961389:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941600.961391:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941600.961394:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c0700 x1860112291496832/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941610 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941600.961400:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941600.961402:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1773941600.961403:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941600.961405:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1773941600.961409:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:2.0:1773941600.961411:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:2.0:1773941600.961412:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:2.0:1773941600.961414:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941600.961416:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:2.0:1773941600.961418:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:2.0:1773941600.961420:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:2.0:1773941600.961424:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941600 v=0 (0 0 0 0) 00000100:00001000:2.0:1773941600.961431:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:2.0:1773941600.961438:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:2.0:1773941600.961440:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941600.961442:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:2.0:1773941600.961444:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941600.961445:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941600.961449:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:2.0:1773941600.961451:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:2.0:1773941600.961453:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:2.0:1773941600.961457:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:2.0:1773941600.961460:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:2.0:1773941600.961462:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941600.961775:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941600.961781:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941600.961789:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c0700 x1860112291496832/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941610 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941600.961799:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291496832:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941600.961805:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941600.961807:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1773941600.961810:0:14804:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000040:3.0:1773941600.961811:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c0700 x1860112291496832/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941610 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941600.961820:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941600.961823:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941600.961826:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a7400. 00000100:00080000:2.0:1773941600.961827:0:14804:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 02000000:00000001:3.0:1773941600.961830:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941600.961833:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941600.961835:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00100000:2.0:1773941600.961835:0:14804:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000020:00000001:3.0:1773941600.961841:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941600.961842:0:14804:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000010:3.0:1773941600.961843:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d11188800. 10000000:01000000:2.0:1773941600.961847:0:14804:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000001:3.0:1773941600.961848:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941600.961852:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000001:00000001:2.0:1773941600.961854:0:14804:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000010:3.0:1773941600.961855:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c0700. 00000100:00000001:3.0:1773941600.961860:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000001:00000001:2.0:1773941600.961860:0:14804:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:3.0:1773941600.961864:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941600.961867:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:2.0:1773941600.961867:0:14804:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:2.0:1773941600.961874:0:14804:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:3.0:1773941600.961876:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000001:00000001:2.0:1773941600.961877:0:14804:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:3.0:1773941600.961881:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:2.0:1773941600.961882:0:14804:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941600.961885:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 10000000:00000001:2.0:1773941600.961886:0:14804:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941600.961889:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941600.961890:0:14804:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:2.0:1773941600.961896:0:14804:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:2.0:1773941600.961899:0:14804:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:2.0:1773941600.961902:0:14804:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:2.0:1773941600.961904:0:14804:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1773941600.961906:0:14804:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:2.0:1773941600.961908:0:14804:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941600.961909:0:14804:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:2.0:1773941600.961910:0:14804:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:2.0:1773941600.961932:0:14804:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:2.0:1773941600.961933:0:14804:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941600.961941:0:14804:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:2.0:1773941600.961944:0:14804:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:2.0:1773941600.961947:0:14804:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941600.961949:0:14804:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941600.961950:0:14804:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941600.961952:0:14804:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941600.961954:0:14804:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.1:1773941600.964999:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d21ec1800. 00010000:00000010:1.1:1773941600.965009:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d00030c00. 00000020:00000001:2.0:1773941601.275185:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941601.275194:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941601.275200:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1773941602.235069:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294992555, at 4294992576 00000100:00000040:1.0:1773941602.235082:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294992555, at 4294992576 00000100:00000010:0.0:1773941602.235107:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580b40. 00000100:00000010:1.0:1773941602.235108:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580ea0. 00000020:00000001:1.0:1773941602.299194:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1773941602.299221:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941602.299227:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941602.555094:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941602.555107:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941602.555116:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941602.555121:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941602.555125:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941602.555128:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941602.555132:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941602.555166:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941602.555170:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:1.0:1773941602.555174:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941602.555177:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941602.555189:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c1f80. 00000020:00000040:1.0:1773941602.555198:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000040:3.0:1773941602.555201:0:461:0:(pinger.c:229:ptlrpc_pinger_process_import()) c1c1ff21-5f68-4bf6-bdc4-258644b24465->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1773941602.555206:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941602.555216:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1773941602.555229:0:461:0:(pinger.c:119:ptlrpc_ping()) Process entered 02000000:00000001:1.0:1773941602.555230:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941602.555275:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d181fc400. 00000100:00000010:3.0:1773941602.555295:0:461:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d111f0a80. 00000100:00000001:1.0:1773941602.555306:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941602.555320:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00000020:00000040:3.0:1773941602.555325:0:461:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941602.555332:0:461:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1773941602.555338:0:461:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 00010000:00000001:1.0:1773941602.555338:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 02000000:00000001:3.0:1773941602.555354:0:461:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:1.0:1773941602.555365:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d04dd4e00. 02000000:00000010:3.0:1773941602.555366:0:461:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff988d08f74500. 00000100:00000001:3.0:1773941602.555384:0:461:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941602.555389:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941602.555398:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d01488000. 00000100:00000040:3.0:1773941602.555406:0:461:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging c1c1ff21-5f68-4bf6-bdc4-258644b24465->MGS req@ffff988d111f0a80 x1860112291497088/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:1.0:1773941602.555410:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941602.555420:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d01488000 with handle 0x8e86767622e2aa9 to hash 00000020:00000001:1.0:1773941602.555433:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941602.555439:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330112180224 : -113743597371392 : ffff988d01488000) 00000100:00000001:3.0:1773941602.555447:0:461:0:(jobid.c:936:lustre_get_jobid()) Process entered 00010000:00000001:1.0:1773941602.555448:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330112180224 : -113743597371392 : ffff988d01488000) 00000100:00000001:3.0:1773941602.555457:0:461:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941602.555461:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941602.555467:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941602.555474:0:461:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000007b933c3a] to pc [ptlrpcd_01_01+1] req@ffff988d111f0a80 x1860112291497088/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00010000:1.0:1773941602.555479:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488000/0x8e86767622e2aa9 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941602.555541:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d01488000/0x8e86767622e2aa9 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941602.555561:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d01488000/0x8e86767622e2aa9 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941602.555571:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941602.555574:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000001:3.0:1773941602.555594:0:461:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941602.555598:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941602.555607:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330528610752 : -113743180940864 : ffff988d1a1ab9c0) 00000100:00000040:3.0:1773941602.555614:0:461:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (330) 00000100:00000001:1.0:1773941602.555614:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941602.555621:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631462784 : -113743078088832 : ffff988d203c1f80) 00000100:00000001:2.0:1773941602.555628:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941602.555632:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941602.555641:0:2404:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941602.555642:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941602.555651:0:2404:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1773941602.555652:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:3.0:1773941602.555661:0:2403:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941602.555661:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:2.0:1773941602.555684:0:2404:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d111f0a80 x1860112291497088/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941602.555685:0:2403:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941602.555685:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c1f80 x1860112291496960/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941602.555759:0:2404:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1773941602.555763:0:2404:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941602.555771:0:2404:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1773941602.555771:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941602.555779:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941602.555781:0:2404:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941602.555789:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941602.555790:0:2404:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941602.555796:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1773941602.555803:0:2404:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d111f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2404:1860112291497088:192.168.204.114@tcp:400:kworker.0 02000000:00000001:1.0:1773941602.555805:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941602.555818:0:2404:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00100000:1.0:1773941602.555820:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c1f80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291496960:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941602.555828:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1773941602.555830:0:2404:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941602.555833:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941602.555837:0:2404:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941602.555839:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941602.555845:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941602.555852:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d1118bc00. 02000000:00000001:2.0:1773941602.555860:0:2404:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1773941602.555861:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941602.555873:0:2404:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff988d03356200. 00000400:00000200:1.0:1773941602.555881:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 02000000:00000001:2.0:1773941602.555883:0:2404:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:1.0:1773941602.555892:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:2.0:1773941602.555894:0:2404:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941602.555905:0:2404:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000400:00000010:1.0:1773941602.555907:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de770. 00000400:00000010:2.0:1773941602.555916:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000200:1.0:1773941602.555922:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291496960, portal 25 00000100:00000200:2.0:1773941602.555927:0:2404:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112291497088, portal 25 00000100:00000001:1.0:1773941602.555929:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941602.555934:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631462784 : -113743078088832 : ffff988d203c1f80) 00000100:00000001:2.0:1773941602.555937:0:2404:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941602.555943:0:2404:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330377898624 : -113743331652992 : ffff988d111f0a80) 00000100:00000040:1.0:1773941602.555949:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c1f80 x1860112291496960/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941618 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941602.555993:0:2404:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d111f0a80 x1860112291497088/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941618 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941602.556030:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1773941602.556039:0:2404:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941602.556046:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000100:00000040:2.0:1773941602.556055:0:2404:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941602.556057:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de198. 00000400:00000010:2.0:1773941602.556066:0:2404:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d04516660. 00000100:00000200:1.0:1773941602.556082:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291496960, offset 0 00000100:00000200:2.0:1773941602.556083:0:2404:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1860112291497088, offset 0 00000400:00000200:1.0:1773941602.556099:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941602.556109:0:2404:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941602.556120:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941602.556129:0:2404:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941602.556139:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941602.556148:0:2404:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941602.556154:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941602.556164:0:2404:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941602.556166:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941602.556184:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941602.556185:0:2404:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941602.556191:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941602.556203:0:2404:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941602.556205:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[160, 159] 00000400:00000200:2.0:1773941602.556218:0:2404:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941602.556222:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:2.0:1773941602.556231:0:2404:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[160, 159] 00000400:00000200:1.0:1773941602.556238:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [271:271:256:4294967295] 192.168.204.124@tcp LPNI seq info [161:161:8:4294967295] 00000400:00000200:2.0:1773941602.556273:0:2404:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:1.0:1773941602.556277:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:2.0:1773941602.556287:0:2404:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [272:272:256:4294967295] 192.168.204.114@tcp LPNI seq info [162:162:8:4294967295] 00000400:00000200:2.0:1773941602.556297:0:2404:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941602.556313:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:1.0:1773941602.556330:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000400:00000200:2.0:1773941602.556344:0:2404:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000010:1.0:1773941602.556348:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfdf00. 00000800:00000200:2.0:1773941602.556358:0:2404:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:2.0:1773941602.556368:0:2404:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0dbb00. 00000800:00000200:1.0:1773941602.556376:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:2.0:1773941602.556385:0:2404:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:1.0:1773941602.556397:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:2.0:1773941602.556405:0:2404:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:1.0:1773941602.556410:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfdf00 type 1, nob 424 niov 1 nkiov 1 00000800:00000200:2.0:1773941602.556414:0:2404:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0dbb00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1773941602.556436:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941602.556447:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941602.556451:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294994152 00000100:00000001:2.0:1773941602.556455:0:2404:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:1.0:1773941602.556459:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3a20. 00000400:00000200:2.0:1773941602.556468:0:2404:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941602.556474:0:2404:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294994152 00000100:00000001:1.0:1773941602.556476:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1773941602.556490:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294994152 00000100:00000001:2.0:1773941602.556492:0:2404:0:(niobuf.c:691:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, keep current deadline 4294994152 00000100:00000010:1.0:1773941602.556496:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3d80. 00000100:00000001:2.0:1773941602.556501:0:2404:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294994152 00000100:00000001:1.0:1773941602.556506:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1773941602.556514:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941602.556515:0:2404:0:(niobuf.c:691:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, keep current deadline 4294994152 00000100:00000001:1.0:1773941602.556521:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941602.556524:0:2404:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941602.556527:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941602.556532:0:2404:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941602.556534:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1773941602.556553:0:2404:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941602.556556:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 11 seconds 00000100:00000001:2.0:1773941602.556561:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941602.556564:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941602.556570:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941602.556575:0:2404:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941602.556580:0:2404:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:2.0:1773941602.556585:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941602.556589:0:2404:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941602.556592:0:2404:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941602.556596:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941602.556634:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941602.556648:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfdf00. 00000400:00000200:0.0:1773941602.556661:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941602.556677:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000800:00000200:1.0:1773941602.556680:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941602.556687:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0dbb00. 00000400:00000200:0.0:1773941602.556691:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de198 00000400:00000200:1.0:1773941602.556694:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000010:0.0:1773941602.556702:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de198. 00000400:00000200:1.0:1773941602.556708:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941602.556717:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04516660 00000100:00000001:0.0:1773941602.556720:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000400:00000010:1.0:1773941602.556721:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d04516660. 00000100:00000001:1.0:1773941602.556733:0:2394:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941602.556742:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c1f80 x1860112291496960/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941618 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:1.0:1773941602.556744:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d111f0a80 x1860112291497088/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941618 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941602.556803:0:2394:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941602.556808:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941602.556811:0:2394:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d111f0a80 x1860112291497088/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941618 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941602.556823:0:2394:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941602.556825:0:2394:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000040:0.0:1773941602.556847:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d203c1f80 x1860112291496960/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941618 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941602.556865:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941602.556869:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:1.0:1773941602.559342:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941602.559363:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941602.559367:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941602.559371:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941602.559381:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941602.559389:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:1.0:1773941602.559400:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941602.559407:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941602.559414:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941602.559420:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324608400 00000400:00000200:1.0:1773941602.559426:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941602.559434:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0x1191 [1] + 224 00000800:00000001:1.0:1773941602.559443:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941602.559454:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941602.559459:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941602.559464:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:1.0:1773941602.559472:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941602.559495:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c1f80 x1860112291496960/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941618 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941602.559510:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d203c1f80 x1860112291496960/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941618 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941602.559548:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1773941602.559563:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941602.559568:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941602.559631:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941602.559639:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941602.559642:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941602.559645:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941602.559651:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941602.559659:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941602.559667:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941602.559673:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941602.559679:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941602.559684:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324608480 00000400:00000200:1.0:1773941602.559689:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941602.559699:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0x1199 [1] + 224 00000100:00000001:0.0:1773941602.559702:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000800:00000001:1.0:1773941602.559707:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000400:00000001:0.0:1773941602.559717:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000800:00000001:1.0:1773941602.559720:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:0.0:1773941602.559723:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000200:1.0:1773941602.559728:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000010:0.0:1773941602.559731:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c5d8. 00000400:00000200:1.0:1773941602.559742:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941602.559747:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de770 00000400:00000010:0.0:1773941602.559753:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de770. 00000100:00000001:0.0:1773941602.559759:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1773941602.559764:0:2393:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941602.559780:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d111f0a80 x1860112291497088/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941618 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941602.559787:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c1f80 x1860112291496960/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941618 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941602.559846:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c1f80 x1860112291496960/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941618 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941602.559853:0:2393:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff988d111f0a80 x1860112291497088/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941618 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941602.559868:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941602.559874:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941602.559883:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941602.559889:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941602.559894:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941602.559897:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941602.559901:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941602.559907:0:2393:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00001000:0.0:1773941602.559911:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=103 v=5 (5 0 0 0) 00000800:00000001:1.0:1773941602.559919:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000100:00001000:0.0:1773941602.559919:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941602.559926:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000800:00000001:1.0:1773941602.559930:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:02020000:0.0:1773941602.559936:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:0.0:1773941602.559941:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941602.559954:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:0.0:1773941602.560005:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:0.0:1773941602.560014:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:0.0:1773941602.560025:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:0.0:1773941602.560047:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:2.0:1773941602.560052:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941602.560053:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00000001:2.0:1773941602.560068:0:2404:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1773941602.560069:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:0.0:1773941602.560084:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000400:00000001:2.0:1773941602.560088:0:2404:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941602.560097:0:2404:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00100000:0.0:1773941602.560099:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d111f0a80 x1860112291497088/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941618 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1773941602.560112:0:2404:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000400:00000200:2.0:1773941602.560125:0:2404:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000100:00100000:0.0:1773941602.560127:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d203c1f80 x1860112291496960/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941618 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000010:2.0:1773941602.560136:0:2404:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000100:00000001:2.0:1773941602.560149:0:2404:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1773941602.560153:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:0.0:1773941602.560157:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000200:2.0:1773941602.560165:0:2404:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d111f0a80 x1860112291497088/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941618 ref 1 fl Rpc:RENQ/200/ffffffff rc -5/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941602.560179:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941602.560186:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:0.0:1773941602.560215:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:0.0:1773941602.560223:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:2.0:1773941602.560228:0:2404:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d111f0a80 x1860112291497088/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941618 ref 1 fl Rpc:RENQU/200/ffffffff rc -5/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941602.560232:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941602.560244:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:2.0:1773941602.560284:0:2404:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941602.560301:0:2404:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 10000000:01000000:0.0:1773941602.560301:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 00000100:00000001:2.0:1773941602.560313:0:2404:0:(client.c:1578:after_reply()) Process entered 10000000:00000001:0.0:1773941602.560314:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941602.560318:0:2404:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:2.0:1773941602.560325:0:2404:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941602.560327:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:2.0:1773941602.560329:0:2404:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941602.560339:0:2404:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941602.560341:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1773941602.560351:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00001000:2.0:1773941602.560353:0:2404:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=103 v=5 (5 0 0 0) 00000100:00001000:2.0:1773941602.560374:0:2404:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941602.560376:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00000001:2.0:1773941602.560380:0:2404:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:2.0:1773941602.560384:0:2404:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941602.560388:0:2404:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:2.0:1773941602.560395:0:2404:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1773941602.560395:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 324 00000100:00000001:2.0:1773941602.560412:0:2404:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941602.560425:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000040:2.0:1773941602.560434:0:2404:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d111f0a80 x1860112291497088/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941618 ref 1 fl Rpc:RENQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000200:0.0:1773941602.560435:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:0.0:1773941602.560464:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941602.560479:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941602.560483:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941602.560487:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:0.0:1773941602.560499:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941602.560505:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:0.0:1773941602.560515:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:2.0:1773941602.560523:0:2404:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000020:00000040:0.0:1773941602.560524:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:0.0:1773941602.560543:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:0.0:1773941602.560548:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:0.0:1773941602.560556:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941602.560558:0:2404:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941602.560564:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1773941602.560568:0:2404:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1773941602.560571:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941602.560579:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000100:00000040:2.0:1773941602.560597:0:2404:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d111f0a80 x1860112291497088/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941618 ref 1 fl Interpret:RENQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1773941602.560611:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:0.0:1773941602.560635:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00100000:2.0:1773941602.560650:0:2404:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d111f0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2404:1860112291497088:192.168.204.114@tcp:400:kworker.0 00000100:00000001:0.0:1773941602.560651:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941602.560666:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1773941602.560671:0:2404:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941602.560676:0:2404:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:2.0:1773941602.560686:0:2404:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d111f0a80 x1860112291497088/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941618 ref 1 fl Complete:RENQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1773941602.560690:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941602.560712:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941602.560728:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941602.560729:0:2404:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1773941602.560743:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941602.560744:0:2404:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941602.560756:0:2404:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff988d03356200. 00000100:00000010:0.0:1773941602.560758:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10835180. 02000000:00000001:2.0:1773941602.560768:0:2404:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000040:0.0:1773941602.560770:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=6 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941602.560780:0:2404:0:(genops.c:1236:class_import_put()) Process entered 02000000:00000001:0.0:1773941602.560783:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 00000020:00000040:2.0:1773941602.560786:0:2404:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:0.0:1773941602.560789:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941602.560800:0:2404:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941602.560802:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000010:2.0:1773941602.560812:0:2404:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff988d08f74500. 02000000:00000001:0.0:1773941602.560813:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941602.560825:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941602.560830:0:2404:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000010:0.0:1773941602.560835:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d02ce6000. 02000000:00000001:2.0:1773941602.560839:0:2404:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1773941602.560844:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:2.0:1773941602.560848:0:2404:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d111f0a80. 00000100:00100000:0.0:1773941602.560859:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d10835180 x1860112291497216/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1773941602.560861:0:2404:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1773941602.560871:0:2404:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941602.560887:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941602.560892:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941602.560899:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941602.560904:0:2404:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:0.0:1773941602.560914:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000052b48ae6] to pc [ptlrpcd_rcv+-1] req@ffff988d10835180 x1860112291497216/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941602.560916:0:2404:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941602.560940:0:2404:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1773941602.560950:0:2404:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941602.561032:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941602.561039:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:0.0:1773941602.561043:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941602.561050:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941602.561055:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941602.561059:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1773941602.561062:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 1 seconds 00000100:00000001:0.0:1773941602.561067:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941602.561072:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1773941602.561081:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c1f80 x1860112291496960/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941618 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941602.561084:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941602.561092:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941602.561098:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1773941602.561099:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:0.0:1773941602.561107:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941602.561113:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10835180 x1860112291497216/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941602.561114:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941602.561145:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941602.561151:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941602.561157:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941602.561160:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c1f80 x1860112291496960/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941618 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1773941602.561165:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941602.561171:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941602.561180:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941602.561195:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c1f80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291496960:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00100000:3.0:1773941602.561196:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10835180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291497216:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941602.561211:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941602.561220:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1773941602.561223:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1773941602.561224:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941602.561237:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1773941602.561239:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000010:3.0:1773941602.561245:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03522400. 00000100:00000001:0.0:1773941602.561268:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1773941602.561276:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1773941602.561279:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941602.561307:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c1f80 x1860112291496960/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941618 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941602.561323:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941602.561333:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00000100:00000001:0.0:1773941602.561334:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1773941602.561344:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1a1ab9c0. 00000400:00000010:3.0:1773941602.561345:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00000100:00000001:0.0:1773941602.561351:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1773941602.561356:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000200:3.0:1773941602.561357:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291497216, portal 25 00010000:00000001:0.0:1773941602.561362:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000001:3.0:1773941602.561364:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941602.561370:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367693184 : -113743341858432 : ffff988d10835180) 00010000:00000001:0.0:1773941602.561371:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1773941602.561376:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941602.561382:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488000 refcount=4 00000100:00000040:3.0:1773941602.561384:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10835180 x1860112291497216/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941612 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1773941602.561390:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112180224 : -113743597371392 : ffff988d01488000) 00010000:00000001:0.0:1773941602.561401:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112180224 : -113743597371392 : ffff988d01488000) 00000100:00000001:3.0:1773941602.561413:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00010000:0.0:1773941602.561415:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d01488000/0x8e86767622e2aa9 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:3.0:1773941602.561424:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:3.0:1773941602.561431:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acbb0. 00010000:00000001:0.0:1773941602.561439:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000200:3.0:1773941602.561440:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291497216, offset 0 00010000:00010000:0.0:1773941602.561454:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d01488000/0x8e86767622e2aa9 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941602.561457:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00000001:0.0:1773941602.561470:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000400:00000200:3.0:1773941602.561471:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:0.0:1773941602.561479:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488000/0x8e86767622e2aa9 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941602.561486:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941602.561499:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941602.561504:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941602.561513:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941602.561523:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941602.561534:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[162, 161] 00010000:00000001:0.0:1773941602.561534:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941602.561540:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:3.0:1773941602.561547:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00010000:00010000:0.0:1773941602.561550:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d01488000/0x8e86767622e2aa9 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941602.561558:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [273:273:256:4294967295] 192.168.204.124@tcp LPNI seq info [163:163:8:4294967295] 00010000:00000001:0.0:1773941602.561578:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:3.0:1773941602.561579:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00010000:00000001:0.0:1773941602.561586:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941602.561593:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:3.0:1773941602.561600:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00010000:00010000:0.0:1773941602.561601:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d01488000/0x8e86767622e2aa9 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941602.561622:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.124@tcp 00010000:00010000:0.0:1773941602.561623:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d01488000 already unused, calling callback (ffffffffc12b4c20) 10000000:00000001:0.0:1773941602.561631:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000800:00000010:3.0:1773941602.561632:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74400. 10000000:00010000:0.0:1773941602.561635:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d01488000/0x8e86767622e2aa9 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941602.561647:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00010000:00000001:0.0:1773941602.561653:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1773941602.561656:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1773941602.561659:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941602.561665:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488000 refcount=5 00000800:00000200:3.0:1773941602.561666:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000020:00000001:0.0:1773941602.561672:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112180224 : -113743597371392 : ffff988d01488000) 00000800:00000200:3.0:1773941602.561675:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74400 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:0.0:1773941602.561679:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112180224 : -113743597371392 : ffff988d01488000) 00010000:00000001:0.0:1773941602.561685:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1773941602.561689:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d01488000/0x8e86767622e2aa9 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1773941602.561697:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000001:3.0:1773941602.561700:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 10000000:00010000:0.0:1773941602.561701:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d01488000/0x8e86767622e2aa9 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941602.561715:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 10000000:01000000:0.0:1773941602.561719:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:0.0:1773941602.561723:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000100:00000001:3.0:1773941602.561732:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294994157 10000000:00000001:0.0:1773941602.561740:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941602.561749:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000100:00000001:3.0:1773941602.561750:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294994152 00010000:00000001:0.0:1773941602.561752:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1773941602.561760:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000100:00000001:3.0:1773941602.561762:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294994157 00010000:00000001:0.0:1773941602.561765:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1773941602.561773:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:3.0:1773941602.561776:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294994152 00010000:00000001:0.0:1773941602.561777:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1773941602.561788:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d01488000 with handle 0x8e86767622e2aa9 from hash 00000100:00000001:3.0:1773941602.561789:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1773941602.561796:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000100:00000001:3.0:1773941602.561798:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941602.561800:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941602.561804:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941602.561806:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:3.0:1773941602.561813:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1773941602.561814:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1773941602.561821:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1773941602.561825:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000100:00000001:3.0:1773941602.561828:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1773941602.561829:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941602.561835:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00010000:00000001:0.0:1773941602.561836:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000200:1.0:1773941602.561838:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00000001:0.0:1773941602.561844:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941602.561845:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000010:1.0:1773941602.561850:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74400. 00000100:00000001:3.0:1773941602.561852:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 10000000:00000001:0.0:1773941602.561854:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941602.561860:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941602.561861:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00010000:0.0:1773941602.561862:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d01488000/0x8e86767622e2aa9 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941602.561869:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941602.561883:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00010000:00000001:0.0:1773941602.561886:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941602.561891:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:1.0:1773941602.561892:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acbb0 00010000:00000001:0.0:1773941602.561895:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000400:00000010:1.0:1773941602.561898:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acbb0. 00010000:00000001:0.0:1773941602.561900:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1773941602.561906:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:1.0:1773941602.561909:0:2394:0:(events.c:34:request_out_callback()) Process entered 00010000:00000001:0.0:1773941602.561910:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941602.561916:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1773941602.561920:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d01488000/0x8e86767622e2aa9 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:1.0:1773941602.561922:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10835180 x1860112291497216/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941612 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000040:0.0:1773941602.561950:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d04dd4e00 count: 0 00000100:00000001:1.0:1773941602.561955:0:2394:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00010000:00000001:0.0:1773941602.561971:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000040:1.0:1773941602.561992:0:2394:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10835180 x1860112291497216/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941612 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941602.562003:0:2394:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941602.562006:0:2394:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:0.0:1773941602.562007:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941602.562014:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941602.562024:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c1f80 x1860112291496960/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941618 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941602.562042:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1773941602.562046:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1773941602.562050:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d1118bc00. 02000000:00000001:0.0:1773941602.562056:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1773941602.562058:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941602.562060:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:0.0:1773941602.562063:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1773941602.562066:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d181fc400. 02000000:00000001:0.0:1773941602.562070:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1773941602.562072:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1773941602.562075:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c1f80. 00000100:00000001:0.0:1773941602.562081:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941602.562083:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1773941602.562085:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:0.0:1773941602.562089:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:0.0:1773941602.562093:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1773941602.562097:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d10872940. 00000020:00000010:0.0:1773941602.562102:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d065c5a00. 00000020:00000010:0.0:1773941602.562110:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d065c5c00. 00000040:00000010:0.0:1773941602.562114:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d065c4200. 00000020:00000010:0.0:1773941602.562120:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d054a7000. 00000020:00000010:0.0:1773941602.562125:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d065c4a00. 80000000:00000010:0.0:1773941602.562131:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d06580b40. 00040000:00000010:0.0:1773941602.562136:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d065c5400. 20000000:00000010:0.0:1773941602.562141:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d065c5e00. 10000000:00000040:0.0:1773941602.562147:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:0.0:1773941602.562150:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:0.0:1773941602.562155:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d065c5c00. 00000040:00000010:0.0:1773941602.562161:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d065c4200. 00000020:00000010:0.0:1773941602.562166:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d054a7000. 00000020:00000010:0.0:1773941602.562171:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d065c4a00. 80000000:00000010:0.0:1773941602.562175:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d06580b40. 00040000:00000010:0.0:1773941602.562180:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d065c5400. 20000000:00000010:0.0:1773941602.562193:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d065c5e00. 00000020:00000010:0.0:1773941602.562198:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d065c5a00. 10000000:00000010:0.0:1773941602.562203:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d10872940. 10000000:01000000:0.0:1773941602.562208:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:0.0:1773941602.562214:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:0.0:1773941602.562219:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:0.0:1773941602.562223:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1773941602.562226:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1773941602.562229:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941602.562234:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:1.0:1773941602.563990:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941602.564001:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941602.564004:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941602.564008:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941602.564015:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941602.564023:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:1.0:1773941602.564031:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941602.564036:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941602.564041:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941602.564045:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324608500 00000400:00000200:1.0:1773941602.564051:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941602.564055:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0x11a9 [1] + 0 00000800:00000001:1.0:1773941602.564061:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941602.564069:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941602.564071:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941602.564076:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:1.0:1773941602.564080:0:2393:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941602.564086:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10835180 x1860112291497216/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941612 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941602.564114:0:2393:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d10835180 x1860112291497216/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941612 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941602.564140:0:2393:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1773941602.564148:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941602.564151:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941602.564216:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941602.564223:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941602.564230:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941602.564233:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941602.564237:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941602.564245:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941602.564271:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941602.564278:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941602.564290:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10835180 x1860112291497216/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941612 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941602.564326:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10835180 x1860112291497216/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941612 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941602.564339:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941602.564342:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941602.564346:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941602.564350:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941602.564354:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941602.564357:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941602.564361:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941602.564369:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=103 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941602.564375:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941602.564392:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941602.564395:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941602.564399:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941602.564402:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941602.564406:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941602.564414:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10835180 x1860112291497216/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941612 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941602.564427:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941602.564432:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941602.564434:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941602.564438:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941602.564447:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941602.564450:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941602.564453:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941602.564457:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941602.564470:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941602.564474:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941602.564479:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941602.564485:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941602 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941602.564491:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941602.564500:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941602.564504:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941602.564507:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941602.564511:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941602.564514:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000400:3.0:1773941602.564519:0:2400:0:(import.c:1275:ptlrpc_connect_interpret()) Evicted from MGS (at 192.168.204.114@tcp) after server handle changed from 0xd03f0241a6d94a04 to 0x349f38f79dc46411 00000100:00080000:3.0:1773941602.572034:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941602.572042:0:2400:0:(import.c:1283:ptlrpc_connect_interpret()) Process leaving via finish (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941602.572047:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941602.572054:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941602.572058:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941602.572428:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941602.572444:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941602.572455:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10835180 x1860112291497216/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941612 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941602.572471:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10835180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291497216:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941602.572473:0:14805:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:3.0:1773941602.572483:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941602.572488:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00080000:1.0:1773941602.572495:0:14805:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00000040:3.0:1773941602.572498:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10835180 x1860112291497216/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941612 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1773941602.572505:0:14805:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:1.0:1773941602.572513:0:14805:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941602.572519:0:14805:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000001:3.0:1773941602.572521:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:1.0:1773941602.572526:0:14805:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:3.0:1773941602.572527:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:1.0:1773941602.572533:0:14805:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000010:3.0:1773941602.572534:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03522400. 00000001:00000001:1.0:1773941602.572541:0:14805:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941602.572542:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941602.572548:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:1.0:1773941602.572550:0:14805:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000040:3.0:1773941602.572553:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000001:00000001:1.0:1773941602.572555:0:14805:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000001:3.0:1773941602.572562:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:1.0:1773941602.572564:0:14805:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941602.572567:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d02ce6000. 10000000:00000001:1.0:1773941602.572570:0:14805:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941602.572576:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1773941602.572577:0:14805:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000001:3.0:1773941602.572583:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:1.0:1773941602.572589:0:14805:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000010:3.0:1773941602.572590:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10835180. 00000100:00000001:3.0:1773941602.572600:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941602.572601:0:14805:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:3.0:1773941602.572604:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941602.572608:0:14805:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941602.572610:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941602.572613:0:14805:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941602.572619:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:1.0:1773941602.572620:0:14805:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:3.0:1773941602.572627:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941602.572629:0:14805:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:3.0:1773941602.572633:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 10000000:01000000:1.0:1773941602.572635:0:14805:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000100:00000001:3.0:1773941602.572638:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:00000040:1.0:1773941602.572641:0:14805:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:1.0:1773941602.572669:0:14805:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941602.572671:0:14805:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941602.572674:0:14805:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941602.572682:0:14805:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:1.0:1773941602.572686:0:14805:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941602.572689:0:14805:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941602.572692:0:14805:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941602.572710:0:14805:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941602.572712:0:14805:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:0.1:1773941602.573065:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d04dd4e00. 00010000:00000010:0.1:1773941602.573081:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d01488000. 00000020:00000001:2.0:1773941603.323162:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941603.323172:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941603.323179:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:0.0:1773941603.771085:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1773941603.771097:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941603.771105:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941603.771110:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941603.771113:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941603.771116:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1773941603.771120:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:0.0:1773941603.771124:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:0.0:1773941603.771128:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:0.0:1773941603.771134:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1773941603.771138:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:0.0:1773941603.771155:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10836a00. 00000020:00000040:0.0:1773941603.771164:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941603.771173:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941603.771179:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941603.771188:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941603.771202:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065c5a00. 00000100:00000001:0.0:1773941603.771213:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941603.771222:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1773941603.771257:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1773941603.771272:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d21ec1400. 00010000:00000001:0.0:1773941603.771283:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1773941603.771289:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d00030400. 00000020:00000001:0.0:1773941603.771296:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:0.0:1773941603.771301:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d00030400 with handle 0x8e86767622e2ab0 to hash 00000020:00000001:0.0:1773941603.771304:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:0.0:1773941603.771327:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00010000:00000001:0.0:1773941603.771332:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00010000:00000001:0.0:1773941603.771337:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941603.771343:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941603.771350:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ab0 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941603.771365:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ab0 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941603.771382:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ab0 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941603.771391:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1773941603.771394:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941603.771402:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb6840. 00000100:00000001:0.0:1773941603.771405:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330139633728 : -113743569917888 : ffff988d02eb6840) 00000100:00000001:0.0:1773941603.771409:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941603.771411:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367699456 : -113743341852160 : ffff988d10836a00) 00000100:00000001:0.0:1773941603.771418:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941603.771423:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941603.771426:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941603.771429:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941603.771443:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10836a00 x1860112291497344/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941603.771456:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941603.771459:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941603.771464:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941603.771468:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941603.771471:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941603.771476:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10836a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291497344:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941603.771482:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941603.771495:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941603.771498:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941603.771502:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941603.771508:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d02ce5800. 02000000:00000001:0.0:1773941603.771511:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941603.771521:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941603.771526:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0014baa8. 00000400:00000010:0.0:1773941603.771534:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d108336e8. 00000100:00000200:0.0:1773941603.771543:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291497344, portal 25 00000100:00000001:0.0:1773941603.771546:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941603.771548:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367699456 : -113743341852160 : ffff988d10836a00) 00000100:00000040:0.0:1773941603.771555:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10836a00 x1860112291497344/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941619 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941603.771566:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941603.771570:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941603.771573:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833d48. 00000100:00000200:0.0:1773941603.771577:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291497344, offset 0 00000400:00000200:0.0:1773941603.771585:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941603.771596:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941603.771605:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941603.771612:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941603.771617:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941603.771623:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941603.771627:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941603.771631:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[162, 163] 00000400:00000200:0.0:1773941603.771637:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:0.0:1773941603.771642:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [274:274:256:4294967295] 192.168.204.114@tcp LPNI seq info [164:164:8:4294967295] 00000400:00000200:0.0:1773941603.771648:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941603.771660:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:0.0:1773941603.771668:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:0.0:1773941603.771673:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d113e9e00. 00000800:00000200:0.0:1773941603.771684:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:0.0:1773941603.771709:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:0.0:1773941603.771718:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d113e9e00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1773941603.771752:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941603.771762:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941603.771768:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294995367 00000100:00000001:0.0:1773941603.771781:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294994157 00000100:00000001:0.0:1773941603.771788:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294995367 00000100:00000001:0.0:1773941603.771794:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294994157 00000100:00000001:0.0:1773941603.771800:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941603.771806:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941603.771811:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941603.771815:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1773941603.771821:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6840 going to sleep for 11 seconds 00000100:00000001:0.0:1773941603.771827:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941603.771833:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941603.772036:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941603.772045:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d113e9e00. 00000400:00000200:1.0:1773941603.772054:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941603.772063:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941603.772070:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833d48 00000400:00000010:1.0:1773941603.772074:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833d48. 00000100:00000001:1.0:1773941603.772079:0:2394:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941603.772087:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10836a00 x1860112291497344/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941619 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941603.772102:0:2394:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941603.772107:0:2394:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d10836a00 x1860112291497344/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941619 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941603.772118:0:2394:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941603.772120:0:2394:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:1.0:1773941603.774163:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941603.774183:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941603.774187:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941603.774190:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941603.774199:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941603.774208:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941603.774218:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941603.774223:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941603.774253:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941603.774259:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324608580 00000400:00000200:1.0:1773941603.774265:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941603.774271:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0x11b1 [1] + 224 00000800:00000001:1.0:1773941603.774280:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941603.774291:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941603.774295:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941603.774299:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:1.0:1773941603.774305:0:2393:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941603.774324:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10836a00 x1860112291497344/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941619 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941603.774339:0:2393:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d10836a00 x1860112291497344/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941619 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941603.774367:0:2393:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1773941603.774379:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941603.774383:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941603.774431:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941603.774441:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941603.774444:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941603.774447:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0014baa8. 00000400:00000200:0.0:1773941603.774453:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d108336e8 00000400:00000010:0.0:1773941603.774456:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d108336e8. 00000100:00000001:0.0:1773941603.774460:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941603.774469:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10836a00 x1860112291497344/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941619 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941603.774486:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10836a00 x1860112291497344/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941619 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941603.774497:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941603.774499:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941603.774503:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941603.774506:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941603.774510:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941603.774514:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941603.774520:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941603.774545:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=104 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941603.774553:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941603.774559:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:0.0:1773941603.774567:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:0.0:1773941603.774573:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941603.774579:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:0.0:1773941603.774585:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:0.0:1773941603.774593:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:0.0:1773941603.774602:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:0.0:1773941603.774609:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:0.0:1773941603.774615:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:0.0:1773941603.774618:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:0.0:1773941603.774622:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:0.0:1773941603.774631:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d10836a00 x1860112291497344/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941619 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941603.774649:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:0.0:1773941603.774651:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:0.0:1773941603.774656:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941603.774661:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:0.0:1773941603.774667:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:0.0:1773941603.774671:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941603.774676:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941603.774678:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941603.774681:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:0.0:1773941603.774684:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941603.774687:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941603.774691:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1773941603.774695:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1773941603.774701:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1773941603.774708:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 325 00000400:00000200:0.0:1773941603.774718:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941603.774726:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:0.0:1773941603.774733:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941603.774748:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941603.774752:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941603.774756:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:0.0:1773941603.774760:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941603.774762:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:0.0:1773941603.774765:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941603.774769:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:0.0:1773941603.774772:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:0.0:1773941603.774776:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:0.0:1773941603.774779:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941603.774781:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941603.774784:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941603.774786:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:0.0:1773941603.774790:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:0.0:1773941603.774794:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:0.0:1773941603.774797:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941603.774800:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1773941603.774805:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941603.774809:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941603.774813:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:0.0:1773941603.774817:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1773941603.774823:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10836300. 00000020:00000040:0.0:1773941603.774829:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:0.0:1773941603.774834:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1773941603.774838:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941603.774842:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941603.774845:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941603.774849:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941603.774857:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d02ce6800. 00000100:00000001:0.0:1773941603.774865:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941603.774876:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d10836300 x1860112291497472/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1773941603.774918:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941603.774924:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941603.774934:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000831309b7] to pc [ptlrpcd_rcv+-1] req@ffff988d10836300 x1860112291497472/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941603.775000:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941603.775005:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:0.0:1773941603.775008:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941603.775013:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941603.775017:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941603.775019:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1773941603.775022:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6840 going to sleep for 1 seconds 00000100:00000001:0.0:1773941603.775027:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941603.775030:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1773941603.775038:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10836a00 x1860112291497344/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941619 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941603.775048:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941603.775049:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:0.0:1773941603.775058:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941603.775060:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941603.775062:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941603.775068:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941603.775070:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941603.775077:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10836a00 x1860112291497344/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941619 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941603.775093:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10836300 x1860112291497472/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941603.775096:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10836a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291497344:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941603.775110:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941603.775117:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941603.775125:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1773941603.775128:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941603.775137:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10836a00 x1860112291497344/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941619 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941603.775139:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941603.775147:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941603.775156:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941603.775160:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000010:0.0:1773941603.775172:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb6840. 00000100:00000001:0.0:1773941603.775176:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1773941603.775183:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:3.0:1773941603.775186:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1773941603.775188:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1773941603.775194:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 02000000:00000001:3.0:1773941603.775198:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941603.775199:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941603.775204:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030400 refcount=4 00000020:00000001:0.0:1773941603.775210:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00000100:00100000:3.0:1773941603.775216:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10836300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291497472:192.168.204.114@tcp:250:ll_cfg_requeue.0 00010000:00000001:0.0:1773941603.775216:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00000100:00000001:3.0:1773941603.775252:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00010000:00010000:0.0:1773941603.775253:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ab0 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941603.775259:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941603.775265:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941603.775274:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:0.0:1773941603.775277:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 02000000:00000010:3.0:1773941603.775283:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03522400. 00010000:00010000:0.0:1773941603.775287:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ab0 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941603.775289:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941603.775307:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:0.0:1773941603.775308:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000400:00000010:3.0:1773941603.775313:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00010000:00010000:0.0:1773941603.775315:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ab0 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:3.0:1773941603.775325:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00010000:00000001:0.0:1773941603.775330:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000200:3.0:1773941603.775334:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291497472, portal 25 00010000:00000001:0.0:1773941603.775335:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1773941603.775342:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ab0 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941603.775343:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941603.775348:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367697664 : -113743341853952 : ffff988d10836300) 00010000:00000001:0.0:1773941603.775366:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000040:3.0:1773941603.775367:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10836300 x1860112291497472/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941613 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1773941603.775373:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941603.775383:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000100:00000001:3.0:1773941603.775392:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00010000:0.0:1773941603.775393:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ab0 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:3.0:1773941603.775405:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:3.0:1773941603.775412:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac220. 00010000:00010000:0.0:1773941603.775415:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d00030400 already unused, calling callback (ffffffffc12b4c20) 00000100:00000200:3.0:1773941603.775419:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291497472, offset 0 00000400:00000200:3.0:1773941603.775429:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 10000000:00000001:0.0:1773941603.775436:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1773941603.775442:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ab0 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941603.775447:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:0.0:1773941603.775461:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000400:00000200:3.0:1773941603.775465:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:0.0:1773941603.775466:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1773941603.775471:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941603.775478:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030400 refcount=5 00000400:00000200:3.0:1773941603.775481:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000020:00000001:0.0:1773941603.775482:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00010000:00000001:0.0:1773941603.775490:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00000400:00000200:3.0:1773941603.775496:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00000001:0.0:1773941603.775498:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1773941603.775506:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ab0 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941603.775509:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941603.775523:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 10000000:00000001:0.0:1773941603.775524:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1773941603.775530:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ab0 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941603.775542:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[164, 163] 10000000:01000000:0.0:1773941603.775545:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:0.0:1773941603.775556:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000400:00000200:3.0:1773941603.775557:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 10000000:00000001:0.0:1773941603.775560:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941603.775568:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000400:00000200:3.0:1773941603.775570:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [275:275:256:4294967295] 192.168.204.124@tcp LPNI seq info [165:165:8:4294967295] 00010000:00000001:0.0:1773941603.775573:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1773941603.775581:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000400:00000200:3.0:1773941603.775586:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00010000:00000001:0.0:1773941603.775587:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1773941603.775592:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941603.775594:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1773941603.775597:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d00030400 with handle 0x8e86767622e2ab0 from hash 00010000:00000001:0.0:1773941603.775605:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1773941603.775609:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941603.775612:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941603.775614:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1773941603.775620:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1773941603.775626:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000400:00000200:3.0:1773941603.775633:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00010000:00000001:0.0:1773941603.775633:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941603.775649:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000200:3.0:1773941603.775652:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.124@tcp 00010000:00000001:0.0:1773941603.775653:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941603.775661:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941603.775667:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ab0 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000010:3.0:1773941603.775670:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74400. 00010000:00000001:0.0:1773941603.775682:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941603.775684:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941603.775689:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000800:00000200:3.0:1773941603.775692:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00010000:00000001:0.0:1773941603.775693:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1773941603.775697:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941603.775701:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941603.775705:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1773941603.775709:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ab0 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941603.775717:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00010000:00000040:0.0:1773941603.775726:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d21ec1400 count: 0 00000800:00000200:3.0:1773941603.775731:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74400 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:0.0:1773941603.775743:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941603.775748:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941603.775751:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1773941603.775760:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941603.775761:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10836a00 x1860112291497344/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941619 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941603.775784:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941603.775786:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1773941603.775794:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1773941603.775796:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294995371 02000000:00000010:0.0:1773941603.775800:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d02ce5800. 02000000:00000001:0.0:1773941603.775808:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1773941603.775813:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294995367 00000020:00000001:0.0:1773941603.775813:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941603.775819:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:0.0:1773941603.775825:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941603.775828:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294995371 02000000:00000010:0.0:1773941603.775830:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065c5a00. 02000000:00000001:0.0:1773941603.775842:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1773941603.775843:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294995367 02000000:00000001:0.0:1773941603.775846:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1773941603.775852:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10836a00. 00000100:00000001:3.0:1773941603.775853:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941603.775861:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941603.775862:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941603.775867:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941603.775870:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941603.775873:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941603.775877:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:01000000:0.0:1773941603.775887:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000100:00000001:3.0:1773941603.775888:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941603.775895:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 10000000:00000001:0.0:1773941603.775897:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000100:00000001:3.0:1773941603.775901:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941603.775906:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 10000000:00000010:0.0:1773941603.775906:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d10872940. 00000100:00000001:3.0:1773941603.775913:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1773941603.775918:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d065c5a00. 00000100:00000001:3.0:1773941603.775920:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1773941603.775927:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d065c5e00. 00000040:00000010:0.0:1773941603.775934:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d065c5400. 00000800:00000200:1.0:1773941603.775941:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000020:00000010:0.0:1773941603.775942:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d054a7000. 00000800:00000010:1.0:1773941603.775952:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74400. 00000020:00000010:0.0:1773941603.775957:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d065c4a00. 00000400:00000200:1.0:1773941603.776006:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941603.776016:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941603.776022:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac220 00000400:00000010:1.0:1773941603.776026:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac220. 00000100:00000001:1.0:1773941603.776033:0:2394:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941603.776045:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10836300 x1860112291497472/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941613 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 80000000:00000010:0.0:1773941603.776047:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d06580b40. 00040000:00000010:0.0:1773941603.776062:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d065c4200. 00000100:00000001:1.0:1773941603.776065:0:2394:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941603.776079:0:2394:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10836300 x1860112291497472/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941613 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 20000000:00000010:0.0:1773941603.776079:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d065c5c00. 10000000:00000040:0.0:1773941603.776096:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 00000100:00000001:1.0:1773941603.776098:0:2394:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941603.776106:0:2394:0:(events.c:68:request_out_callback()) Process leaving 10000000:00000001:0.0:1773941603.776110:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:0.0:1773941603.776124:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d065c5e00. 00000040:00000010:0.0:1773941603.776134:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d065c5400. 00000020:00000010:0.0:1773941603.776143:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d054a7000. 00000020:00000010:0.0:1773941603.776152:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d065c4a00. 80000000:00000010:0.0:1773941603.776158:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d06580b40. 00040000:00000010:0.0:1773941603.776164:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d065c4200. 20000000:00000010:0.0:1773941603.776169:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d065c5c00. 00000020:00000010:0.0:1773941603.776174:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d065c5a00. 10000000:00000010:0.0:1773941603.776178:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d10872940. 10000000:01000000:0.0:1773941603.776183:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:0.0:1773941603.776187:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:0.0:1773941603.776191:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:0.0:1773941603.776194:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1773941603.776196:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1773941603.776198:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941603.776201:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00010000:00000010:0.1:1773941603.781061:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d21ec1400. 00010000:00000010:0.1:1773941603.781076:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d00030400. 00000800:00000001:1.0:1773941603.783757:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941603.783770:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941603.783773:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941603.783776:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941603.783783:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941603.783789:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:1.0:1773941603.783798:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941603.783802:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941603.783807:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941603.783811:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324608600 00000400:00000200:1.0:1773941603.783817:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941603.783834:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0x11c1 [1] + 0 00000800:00000001:1.0:1773941603.783842:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941603.783851:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941603.783855:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941603.783859:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:1.0:1773941603.783865:0:2393:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941603.783872:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10836300 x1860112291497472/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941613 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941603.783887:0:2393:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d10836300 x1860112291497472/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941613 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941603.783912:0:2393:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1773941603.783921:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941603.783925:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941603.783957:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941603.784003:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941603.784012:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941603.784015:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941603.784020:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941603.784031:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941603.784034:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941603.784040:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941603.784052:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10836300 x1860112291497472/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941613 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941603.784074:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10836300 x1860112291497472/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941613 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941603.784105:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941603.784109:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941603.784113:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941603.784117:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941603.784120:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941603.784124:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941603.784129:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941603.784135:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=104 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941603.784142:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941603.784148:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941603.784152:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941603.784157:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941603.784160:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941603.784166:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941603.784176:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10836300 x1860112291497472/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941613 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941603.784193:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941603.784197:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941603.784200:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941603.784204:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941603.784213:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941603.784217:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941603.784220:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941603.784223:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941603.784247:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941603.784251:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941603.784255:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941603.784260:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941603 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941603.784264:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941603.784272:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941603.784274:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941603.784277:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941603.784279:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941603.784292:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941603.784297:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941603.784300:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941603.784304:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941603.784308:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941603.784313:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941603.784316:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941603.784691:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941603.784697:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941603.784705:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10836300 x1860112291497472/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941613 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941603.784719:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10836300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291497472:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941603.784728:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941603.784730:0:14806:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:3.0:1773941603.784732:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941603.784740:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10836300 x1860112291497472/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941613 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1773941603.784748:0:14806:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00000001:3.0:1773941603.784762:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00100000:1.0:1773941603.784763:0:14806:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000001:3.0:1773941603.784768:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:1.0:1773941603.784770:0:14806:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941603.784777:0:14806:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000010:3.0:1773941603.784778:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03522400. 00000001:00000001:1.0:1773941603.784782:0:14806:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:3.0:1773941603.784786:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:1.0:1773941603.784788:0:14806:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000001:3.0:1773941603.784792:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:1.0:1773941603.784795:0:14806:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:3.0:1773941603.784797:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000001:00000001:1.0:1773941603.784803:0:14806:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:3.0:1773941603.784806:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:1.0:1773941603.784808:0:14806:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000010:3.0:1773941603.784812:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d02ce6800. 00000001:00000001:1.0:1773941603.784814:0:14806:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941603.784822:0:14806:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941603.784823:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1773941603.784826:0:14806:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000001:3.0:1773941603.784829:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:1.0:1773941603.784834:0:14806:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000010:3.0:1773941603.784835:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10836300. 00000100:00000001:1.0:1773941603.784847:0:14806:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:3.0:1773941603.784848:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941603.784853:0:14806:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941603.784854:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941603.784861:0:14806:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941603.784862:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1773941603.784870:0:14806:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:3.0:1773941603.784875:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941603.784876:0:14806:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:3.0:1773941603.784881:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 10000000:01000000:1.0:1773941603.784882:0:14806:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941603.784886:0:14806:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00000001:3.0:1773941603.784897:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941603.784903:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941603.784922:0:14806:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941603.784927:0:14806:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941603.784930:0:14806:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941603.784935:0:14806:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:1.0:1773941603.784941:0:14806:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941603.784944:0:14806:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941603.784947:0:14806:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941603.784951:0:14806:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941603.784954:0:14806:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941604.347207:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1773941604.347267:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941604.347274:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1773941605.051121:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294995371, at 4294995392 00000100:00000040:1.0:1773941605.051123:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294995371, at 4294995392 00000100:00000010:1.0:1773941605.051143:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3d80. 00000100:00000010:0.0:1773941605.051149:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3a20. 10000000:00000001:0.0:1773941605.179066:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1773941605.179075:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941605.179103:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941605.179107:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941605.179110:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941605.179112:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1773941605.179115:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:0.0:1773941605.179118:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:0.0:1773941605.179122:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:0.0:1773941605.179126:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1773941605.179128:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:0.0:1773941605.179139:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10836a00. 00000020:00000040:0.0:1773941605.179145:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941605.179152:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941605.179156:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941605.179163:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941605.179172:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065c5a00. 00000100:00000001:0.0:1773941605.179182:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941605.179188:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1773941605.179193:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1773941605.179228:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d21ec1400. 00010000:00000001:0.0:1773941605.179236:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1773941605.179240:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d00030400. 00000020:00000001:0.0:1773941605.179245:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:0.0:1773941605.179249:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d00030400 with handle 0x8e86767622e2ab7 to hash 00000020:00000001:0.0:1773941605.179252:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:0.0:1773941605.179255:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00010000:00000001:0.0:1773941605.179260:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00010000:00000001:0.0:1773941605.179265:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941605.179270:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941605.179275:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ab7 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941605.179290:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ab7 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941605.179307:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ab7 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941605.179343:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1773941605.179346:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941605.179354:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb6840. 00000100:00000001:0.0:1773941605.179357:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330139633728 : -113743569917888 : ffff988d02eb6840) 00000100:00000001:0.0:1773941605.179360:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941605.179363:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367699456 : -113743341852160 : ffff988d10836a00) 00000100:00000001:0.0:1773941605.179369:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941605.179375:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941605.179378:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941605.179380:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941605.179394:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10836a00 x1860112291497600/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941605.179407:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941605.179410:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941605.179415:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941605.179419:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941605.179422:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941605.179427:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10836a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291497600:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941605.179433:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941605.179440:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941605.179443:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941605.179447:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941605.179453:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d02ce6c00. 02000000:00000001:0.0:1773941605.179456:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941605.179465:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941605.179470:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0014baa8. 00000400:00000010:0.0:1773941605.179477:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d108336e8. 00000100:00000200:0.0:1773941605.179486:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291497600, portal 25 00000100:00000001:0.0:1773941605.179488:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941605.179491:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367699456 : -113743341852160 : ffff988d10836a00) 00000100:00000040:0.0:1773941605.179497:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10836a00 x1860112291497600/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941621 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941605.179522:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941605.179527:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941605.179531:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833e58. 00000100:00000200:0.0:1773941605.179535:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291497600, offset 0 00000400:00000200:0.0:1773941605.179543:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941605.179553:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941605.179563:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941605.179570:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941605.179576:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941605.179581:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941605.179585:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941605.179590:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[164, 165] 00000400:00000200:0.0:1773941605.179595:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:0.0:1773941605.179600:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [276:276:256:4294967295] 192.168.204.114@tcp LPNI seq info [166:166:8:4294967295] 00000400:00000200:0.0:1773941605.179606:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941605.179619:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:0.0:1773941605.179626:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:0.0:1773941605.179632:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d113e9500. 00000800:00000200:0.0:1773941605.179641:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:0.0:1773941605.179651:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:0.0:1773941605.179657:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d113e9500 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1773941605.179683:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941605.179696:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941605.179700:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294996775 00000100:00000010:0.0:1773941605.179706:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580b40. 00000100:00000001:0.0:1773941605.179715:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1773941605.179725:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294996775 00000100:00000010:0.0:1773941605.179730:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580e40. 00000100:00000001:0.0:1773941605.179735:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1773941605.179751:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941605.179755:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941605.179758:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941605.179762:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1773941605.179766:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6840 going to sleep for 11 seconds 00000100:00000001:0.0:1773941605.179770:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941605.179774:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941605.179924:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941605.179935:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d113e9500. 00000400:00000200:1.0:1773941605.179946:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941605.179957:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941605.180012:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833e58 00000400:00000010:1.0:1773941605.180019:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833e58. 00000100:00000001:1.0:1773941605.180026:0:2394:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941605.180038:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10836a00 x1860112291497600/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941621 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941605.180058:0:2394:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941605.180066:0:2394:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d10836a00 x1860112291497600/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941621 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941605.180082:0:2394:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941605.180086:0:2394:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:1.0:1773941605.182142:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941605.182162:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941605.182166:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941605.182170:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941605.182177:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941605.182185:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941605.182230:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941605.182236:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941605.182243:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941605.182248:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324608680 00000400:00000200:1.0:1773941605.182254:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941605.182261:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0x11c9 [1] + 224 00000800:00000001:1.0:1773941605.182270:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941605.182284:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941605.182288:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941605.182294:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:1.0:1773941605.182302:0:2393:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941605.182310:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10836a00 x1860112291497600/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941621 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941605.182325:0:2393:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d10836a00 x1860112291497600/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941621 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941605.182357:0:2393:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1773941605.182368:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941605.182373:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941605.182447:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941605.182457:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941605.182460:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941605.182464:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0014baa8. 00000400:00000200:0.0:1773941605.182472:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d108336e8 00000400:00000010:0.0:1773941605.182474:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d108336e8. 00000100:00000001:0.0:1773941605.182478:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941605.182488:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10836a00 x1860112291497600/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941621 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941605.182504:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10836a00 x1860112291497600/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941621 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941605.182514:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941605.182517:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941605.182521:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941605.182524:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941605.182528:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941605.182530:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941605.182534:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941605.182541:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=106 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941605.182546:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941605.182550:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:0.0:1773941605.182555:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:0.0:1773941605.182560:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941605.182564:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:0.0:1773941605.182568:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:0.0:1773941605.182573:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:0.0:1773941605.182579:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:0.0:1773941605.182596:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:0.0:1773941605.182600:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:0.0:1773941605.182603:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:0.0:1773941605.182606:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:0.0:1773941605.182611:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d10836a00 x1860112291497600/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941621 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941605.182621:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:0.0:1773941605.182623:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:0.0:1773941605.182626:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941605.182630:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:0.0:1773941605.182633:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:0.0:1773941605.182636:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941605.182639:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941605.182640:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941605.182642:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:0.0:1773941605.182644:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941605.182646:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941605.182649:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1773941605.182652:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1773941605.182656:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1773941605.182659:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 326 00000400:00000200:0.0:1773941605.182666:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941605.182672:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:0.0:1773941605.182677:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941605.182681:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941605.182684:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941605.182687:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:0.0:1773941605.182692:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941605.182694:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:0.0:1773941605.182697:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941605.182701:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:0.0:1773941605.182704:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:0.0:1773941605.182707:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:0.0:1773941605.182719:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941605.182724:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941605.182728:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941605.182732:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:0.0:1773941605.182738:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:0.0:1773941605.182745:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:0.0:1773941605.182751:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941605.182756:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1773941605.182763:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941605.182768:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941605.182773:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:0.0:1773941605.182777:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1773941605.182785:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10837480. 00000020:00000040:0.0:1773941605.182791:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:0.0:1773941605.182797:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1773941605.182801:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941605.182805:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941605.182809:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941605.182813:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941605.182823:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d02ce4800. 00000100:00000001:0.0:1773941605.182830:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941605.182841:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d10837480 x1860112291497728/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1773941605.182852:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941605.182856:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941605.182862:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084c3e071] to pc [ptlrpcd_rcv+-1] req@ffff988d10837480 x1860112291497728/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941605.182893:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941605.182897:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:0.0:1773941605.182899:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941605.182903:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941605.182907:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941605.182917:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1773941605.182920:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6840 going to sleep for 1 seconds 00000100:00000001:0.0:1773941605.182923:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941605.182926:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1773941605.182933:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10836a00 x1860112291497600/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941621 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941605.182944:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:0.0:1773941605.182948:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941605.182951:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941605.182954:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1773941605.182997:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10836a00 x1860112291497600/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941621 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941605.183010:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10836a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291497600:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941605.183017:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941605.183019:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941605.183022:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1773941605.183025:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941605.183029:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10836a00 x1860112291497600/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941621 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941605.183039:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1773941605.183043:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb6840. 00000100:00000001:0.0:1773941605.183049:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1773941605.183051:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:0.0:1773941605.183055:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1773941605.183059:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1773941605.183067:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:3.0:1773941605.183068:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000040:0.0:1773941605.183074:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030400 refcount=4 00000020:00000001:0.0:1773941605.183083:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00000100:00000001:3.0:1773941605.183085:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1773941605.183099:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00000100:00000001:3.0:1773941605.183101:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00010000:00010000:0.0:1773941605.183116:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ab7 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:3.0:1773941605.183132:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10837480 x1860112291497728/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1773941605.183166:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941605.183187:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00010000:0.0:1773941605.183188:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ab7 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941605.183194:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941605.183245:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 02000000:00000001:3.0:1773941605.183246:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00010000:00010000:0.0:1773941605.183252:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ab7 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941605.183254:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941605.183273:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941605.183295:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00100000:3.0:1773941605.183298:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10837480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291497728:192.168.204.114@tcp:250:ll_cfg_requeue.0 00010000:00000001:0.0:1773941605.183299:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1773941605.183311:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ab7 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941605.183313:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941605.183327:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00000001:0.0:1773941605.183328:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 02000000:00000001:3.0:1773941605.183337:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941605.183338:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941605.183358:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:0.0:1773941605.183359:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1773941605.183375:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ab7 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000010:3.0:1773941605.183380:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03522400. 02000000:00000001:3.0:1773941605.183393:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941605.183395:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d00030400 already unused, calling callback (ffffffffc12b4c20) 10000000:00000001:0.0:1773941605.183408:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1773941605.183412:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ab7 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941605.183420:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941605.183437:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00010000:00000001:0.0:1773941605.183441:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1773941605.183446:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1773941605.183449:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941605.183455:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030400 refcount=5 00000400:00000010:3.0:1773941605.183457:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00000020:00000001:0.0:1773941605.183473:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00010000:00000001:0.0:1773941605.183478:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00000100:00000200:3.0:1773941605.183480:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291497728, portal 25 00010000:00000001:0.0:1773941605.183484:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000100:00000001:3.0:1773941605.183486:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941605.183489:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367702144 : -113743341849472 : ffff988d10837480) 00010000:00010000:0.0:1773941605.183500:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ab7 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:3.0:1773941605.183502:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10837480 x1860112291497728/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941615 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000001:0.0:1773941605.183561:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000001:3.0:1773941605.183565:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 10000000:00010000:0.0:1773941605.183567:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ab7 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:3.0:1773941605.183583:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:3.0:1773941605.183594:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac110. 10000000:01000000:0.0:1773941605.183600:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000100:00000200:3.0:1773941605.183605:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291497728, offset 0 10000000:00000040:0.0:1773941605.183607:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1773941605.183614:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941605.183620:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000400:00000200:3.0:1773941605.183621:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00000001:0.0:1773941605.183629:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1773941605.183635:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1773941605.183638:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000400:00000200:3.0:1773941605.183647:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:0.0:1773941605.183647:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941605.183660:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941605.183669:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000020:00000040:0.0:1773941605.183671:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d00030400 with handle 0x8e86767622e2ab7 from hash 00010000:00000001:0.0:1773941605.183677:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1773941605.183679:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941605.183681:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941605.183682:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1773941605.183684:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1773941605.183687:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1773941605.183690:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941605.183691:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941605.183693:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941605.183695:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941605.183722:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00010000:0.0:1773941605.183723:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ab7 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941605.183736:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941605.183747:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941605.183757:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941605.183773:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[166, 165] 00010000:00000001:0.0:1773941605.183789:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:3.0:1773941605.183790:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00010000:00000001:0.0:1773941605.183800:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:3.0:1773941605.183807:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [277:277:256:4294967295] 192.168.204.124@tcp LPNI seq info [167:167:8:4294967295] 00010000:00000001:0.0:1773941605.183808:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1773941605.183816:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1773941605.183827:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:3.0:1773941605.183829:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00010000:00000001:0.0:1773941605.183834:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941605.183852:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:3.0:1773941605.183862:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00010000:00010000:0.0:1773941605.183862:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ab7 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1773941605.183897:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d21ec1400 count: 0 00000800:00000200:3.0:1773941605.183910:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.124@tcp 00010000:00000001:0.0:1773941605.183923:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000010:3.0:1773941605.183926:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74500. 00010000:00000001:0.0:1773941605.183929:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941605.183946:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000800:00000200:3.0:1773941605.184013:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000100:00000040:0.0:1773941605.184014:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10836a00 x1860112291497600/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941621 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1773941605.184039:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000100:00000001:0.0:1773941605.184039:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1773941605.184052:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000800:00000200:3.0:1773941605.184054:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74500 type 1, nob 616 niov 1 nkiov 1 02000000:00000010:0.0:1773941605.184059:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d02ce6c00. 02000000:00000001:0.0:1773941605.184070:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1773941605.184074:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941605.184077:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:0.0:1773941605.184082:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941605.184086:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941605.184088:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065c5a00. 02000000:00000001:0.0:1773941605.184104:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000400:00000200:3.0:1773941605.184107:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 02000000:00000001:0.0:1773941605.184109:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1773941605.184114:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294996780 00000100:00000010:0.0:1773941605.184115:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10836a00. 00000100:00000001:0.0:1773941605.184136:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941605.184140:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941605.184144:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294996775 10000000:00000001:0.0:1773941605.184145:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:0.0:1773941605.184163:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000100:00000001:3.0:1773941605.184171:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294996780 10000000:00000001:0.0:1773941605.184172:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1773941605.184185:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d10872940. 00000100:00000001:3.0:1773941605.184188:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294996775 00000800:00000200:1.0:1773941605.184197:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000020:00000010:0.0:1773941605.184225:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d065c5a00. 00000100:00000001:3.0:1773941605.184228:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000010:1.0:1773941605.184241:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74500. 00000100:00000001:3.0:1773941605.184243:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1773941605.184246:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d065c5c00. 00000100:00000001:3.0:1773941605.184265:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000040:00000010:0.0:1773941605.184266:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d065c4200. 00000400:00000200:1.0:1773941605.184268:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1773941605.184288:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:0.0:1773941605.184290:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d054a7000. 00000020:00000010:0.0:1773941605.184309:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d065c4a00. 00000400:00000200:1.0:1773941605.184310:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 80000000:00000010:0.0:1773941605.184322:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d06580d80. 00000400:00000200:1.0:1773941605.184327:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac110 00040000:00000010:0.0:1773941605.184331:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d065c5400. 00000400:00000010:1.0:1773941605.184339:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac110. 20000000:00000010:0.0:1773941605.184340:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d065c5e00. 00000100:00000001:1.0:1773941605.184350:0:2394:0:(events.c:34:request_out_callback()) Process entered 10000000:00000040:0.0:1773941605.184364:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 00000100:00000200:1.0:1773941605.184368:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10837480 x1860112291497728/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941615 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000001:0.0:1773941605.184371:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:0.0:1773941605.184385:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d065c5c00. 00000040:00000010:0.0:1773941605.184398:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d065c4200. 00000100:00000001:1.0:1773941605.184406:0:2394:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000020:00000010:0.0:1773941605.184409:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d054a7000. 00000020:00000010:0.0:1773941605.184419:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d065c4a00. 00000100:00000040:1.0:1773941605.184422:0:2394:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10837480 x1860112291497728/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941615 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 80000000:00000010:0.0:1773941605.184427:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d06580d80. 00040000:00000010:0.0:1773941605.184437:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d065c5400. 00000100:00000001:3.0:1773941605.184442:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 20000000:00000010:0.0:1773941605.184449:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d065c5e00. 00000100:00000001:3.0:1773941605.184455:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000020:00000010:0.0:1773941605.184458:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d065c5a00. 00000100:00000001:3.0:1773941605.184479:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 10000000:00000010:0.0:1773941605.184480:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d10872940. 00000100:00000001:1.0:1773941605.184481:0:2394:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941605.184491:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 10000000:01000000:0.0:1773941605.184492:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 00000100:00000001:3.0:1773941605.184503:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941605.184505:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:1.0:1773941605.184507:0:2394:0:(events.c:68:request_out_callback()) Process leaving 10000000:00020000:0.0:1773941605.184522:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 00000100:00000001:3.0:1773941605.184524:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941605.184530:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1773941605.184538:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1773941605.184541:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941605.184545:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00010000:00000010:0.1:1773941605.189015:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d21ec1400. 00010000:00000010:0.1:1773941605.189024:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d00030400. 00000800:00000001:1.0:1773941605.191406:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941605.191423:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941605.191428:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941605.191434:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941605.191443:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941605.191454:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:1.0:1773941605.191465:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941605.191472:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941605.191480:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941605.191486:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324608700 00000400:00000200:1.0:1773941605.191495:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941605.191504:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0x11d9 [1] + 0 00000800:00000001:1.0:1773941605.191516:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941605.191529:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941605.191534:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941605.191542:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:1.0:1773941605.191552:0:2393:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941605.191565:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10837480 x1860112291497728/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941615 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941605.191589:0:2393:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d10837480 x1860112291497728/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941615 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941605.191627:0:2393:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1773941605.191640:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941605.191649:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941605.191676:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941605.191694:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941605.191703:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941605.191705:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941605.191710:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941605.191717:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941605.191721:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941605.191726:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941605.191736:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10837480 x1860112291497728/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941615 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941605.191753:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10837480 x1860112291497728/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941615 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941605.191763:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941605.191765:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941605.191769:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941605.191773:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941605.191776:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941605.191779:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941605.191783:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941605.191791:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=106 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941605.191797:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941605.191801:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941605.191803:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941605.191808:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941605.191810:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941605.191814:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941605.191820:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10837480 x1860112291497728/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941615 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941605.191831:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941605.191834:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941605.191837:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941605.191841:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941605.191849:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941605.191853:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941605.191856:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941605.191859:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941605.191871:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941605.191874:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941605.191878:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941605.191884:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941605 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941605.191888:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941605.191898:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941605.191902:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941605.191907:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941605.191910:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941605.191912:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941605.191918:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941605.191921:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941605.191926:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941605.191931:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941605.191935:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941605.191938:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941605.192366:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941605.192372:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941605.192381:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10837480 x1860112291497728/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941615 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941605.192394:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10837480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291497728:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941605.192402:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941605.192406:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941605.192412:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10837480 x1860112291497728/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941615 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941605.192423:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941605.192428:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941605.192433:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03522400. 02000000:00000001:3.0:1773941605.192438:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:1.0:1773941605.192439:0:14807:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000020:00000001:3.0:1773941605.192442:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941605.192457:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00080000:1.0:1773941605.192467:0:14807:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000020:00000001:3.0:1773941605.192468:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941605.192479:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d02ce4800. 00000100:00100000:1.0:1773941605.192483:0:14807:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000001:3.0:1773941605.192487:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1773941605.192493:0:14807:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:3.0:1773941605.192494:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 10000000:01000000:1.0:1773941605.192509:0:14807:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000010:3.0:1773941605.192510:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10837480. 00000001:00000001:1.0:1773941605.192529:0:14807:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:3.0:1773941605.192531:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941605.192540:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941605.192544:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941605.192550:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000001:00000001:1.0:1773941605.192552:0:14807:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:3.0:1773941605.192554:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941605.192564:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000001:00000001:1.0:1773941605.192566:0:14807:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941605.192568:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941605.192590:0:14807:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941605.192593:0:14807:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941605.192597:0:14807:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941605.192601:0:14807:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941605.192604:0:14807:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:1.0:1773941605.192613:0:14807:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:1.0:1773941605.192618:0:14807:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941605.192623:0:14807:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941605.192626:0:14807:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941605.192630:0:14807:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941605.192634:0:14807:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941605.192637:0:14807:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941605.192640:0:14807:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:1.0:1773941605.192668:0:14807:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941605.192671:0:14807:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941605.192675:0:14807:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941605.192681:0:14807:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:1.0:1773941605.192687:0:14807:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941605.192693:0:14807:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941605.192698:0:14807:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941605.192704:0:14807:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941605.192707:0:14807:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941605.371146:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941605.371168:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941605.371172:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:1.0:1773941606.395250:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1773941606.395255:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941606.395259:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941606.459106:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294996780, at 4294996800 00000100:00000040:0.0:1773941606.459107:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294996780, at 4294996800 00000100:00000010:0.0:1773941606.459122:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580b40. 00000100:00000010:1.0:1773941606.459125:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580e40. 10000000:00000001:0.0:1773941606.523071:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1773941606.523078:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941606.523081:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941606.523083:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941606.523085:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941606.523086:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1773941606.523089:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:0.0:1773941606.523091:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:0.0:1773941606.523093:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:0.0:1773941606.523096:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1773941606.523097:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:0.0:1773941606.523107:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10836a00. 00000020:00000040:0.0:1773941606.523111:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941606.523115:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941606.523145:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941606.523151:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941606.523158:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065c5a00. 00000100:00000001:0.0:1773941606.523165:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941606.523169:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1773941606.523172:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1773941606.523193:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d21ec1400. 00010000:00000001:0.0:1773941606.523199:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1773941606.523202:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d00030400. 00000020:00000001:0.0:1773941606.523205:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:0.0:1773941606.523207:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d00030400 with handle 0x8e86767622e2abe to hash 00000020:00000001:0.0:1773941606.523209:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:0.0:1773941606.523210:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00010000:00000001:0.0:1773941606.523213:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00010000:00000001:0.0:1773941606.523216:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941606.523219:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941606.523223:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2abe lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941606.523232:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2abe lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941606.523242:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2abe lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941606.523247:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1773941606.523249:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941606.523254:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb6840. 00000100:00000001:0.0:1773941606.523256:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330139633728 : -113743569917888 : ffff988d02eb6840) 00000100:00000001:0.0:1773941606.523258:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941606.523260:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367699456 : -113743341852160 : ffff988d10836a00) 00000100:00000001:0.0:1773941606.523264:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941606.523268:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941606.523270:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941606.523276:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941606.523285:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10836a00 x1860112291497856/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941606.523294:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941606.523296:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941606.523299:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941606.523301:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941606.523303:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941606.523306:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10836a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291497856:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941606.523310:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941606.523314:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941606.523316:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941606.523318:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941606.523322:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d02ce6c00. 02000000:00000001:0.0:1773941606.523324:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941606.523331:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941606.523335:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0014baa8. 00000400:00000010:0.0:1773941606.523340:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d108336e8. 00000100:00000200:0.0:1773941606.523346:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291497856, portal 25 00000100:00000001:0.0:1773941606.523348:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941606.523349:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367699456 : -113743341852160 : ffff988d10836a00) 00000100:00000040:0.0:1773941606.523353:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10836a00 x1860112291497856/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941622 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941606.523360:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941606.523362:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941606.523364:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833f68. 00000100:00000200:0.0:1773941606.523367:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291497856, offset 0 00000400:00000200:0.0:1773941606.523372:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941606.523379:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941606.523385:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941606.523390:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941606.523393:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941606.523397:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941606.523403:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941606.523406:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[166, 167] 00000400:00000200:0.0:1773941606.523409:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:0.0:1773941606.523413:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [278:278:256:4294967295] 192.168.204.114@tcp LPNI seq info [168:168:8:4294967295] 00000400:00000200:0.0:1773941606.523416:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941606.523425:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:0.0:1773941606.523430:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:0.0:1773941606.523434:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d113e9600. 00000800:00000200:0.0:1773941606.523440:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:0.0:1773941606.523447:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:0.0:1773941606.523451:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d113e9600 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1773941606.523468:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941606.523471:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941606.523473:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294998119 00000100:00000010:0.0:1773941606.523476:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580b40. 00000100:00000001:0.0:1773941606.523482:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1773941606.523487:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294998119 00000100:00000010:0.0:1773941606.523490:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580d80. 00000100:00000001:0.0:1773941606.523493:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1773941606.523496:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941606.523499:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941606.523501:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941606.523503:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1773941606.523505:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6840 going to sleep for 11 seconds 00000100:00000001:0.0:1773941606.523508:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941606.523511:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941606.523608:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941606.523614:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d113e9600. 00000400:00000200:1.0:1773941606.523620:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941606.523625:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941606.523640:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833f68 00000400:00000010:1.0:1773941606.523643:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833f68. 00000100:00000001:1.0:1773941606.523646:0:2394:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941606.523651:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10836a00 x1860112291497856/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941622 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941606.523666:0:2394:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941606.523670:0:2394:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d10836a00 x1860112291497856/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941622 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941606.523676:0:2394:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941606.523677:0:2394:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:1.0:1773941606.525869:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941606.525891:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941606.525895:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941606.525898:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941606.525906:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941606.525914:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941606.525922:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941606.525927:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941606.525933:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941606.525938:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324608780 00000400:00000200:1.0:1773941606.525944:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941606.525950:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0x11e1 [1] + 224 00000800:00000001:1.0:1773941606.526004:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941606.526016:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941606.526020:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941606.526025:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:1.0:1773941606.526030:0:2393:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941606.526038:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10836a00 x1860112291497856/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941622 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941606.526052:0:2393:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d10836a00 x1860112291497856/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941622 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941606.526077:0:2393:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1773941606.526088:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941606.526092:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941606.526234:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941606.526241:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941606.526242:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941606.526245:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0014baa8. 00000400:00000200:0.0:1773941606.526249:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d108336e8 00000400:00000010:0.0:1773941606.526250:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d108336e8. 00000100:00000001:0.0:1773941606.526259:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941606.526265:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10836a00 x1860112291497856/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941622 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941606.526274:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10836a00 x1860112291497856/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941622 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941606.526280:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941606.526282:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941606.526284:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941606.526286:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941606.526288:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941606.526290:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941606.526292:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941606.526297:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=107 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941606.526300:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941606.526302:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:0.0:1773941606.526306:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:0.0:1773941606.526308:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941606.526311:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:0.0:1773941606.526313:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:0.0:1773941606.526317:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:0.0:1773941606.526320:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:0.0:1773941606.526323:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:0.0:1773941606.526325:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:0.0:1773941606.526327:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:0.0:1773941606.526328:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:0.0:1773941606.526331:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d10836a00 x1860112291497856/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941622 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941606.526337:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:0.0:1773941606.526338:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:0.0:1773941606.526340:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941606.526342:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:0.0:1773941606.526344:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:0.0:1773941606.526349:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941606.526351:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941606.526352:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941606.526353:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:0.0:1773941606.526354:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941606.526356:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941606.526358:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1773941606.526359:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1773941606.526362:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1773941606.526364:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 328 00000400:00000200:0.0:1773941606.526368:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941606.526371:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:0.0:1773941606.526374:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941606.526377:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941606.526379:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941606.526381:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:0.0:1773941606.526383:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941606.526385:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:0.0:1773941606.526387:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941606.526389:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:0.0:1773941606.526391:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:0.0:1773941606.526393:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:0.0:1773941606.526395:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941606.526396:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941606.526397:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941606.526399:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:0.0:1773941606.526401:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:0.0:1773941606.526403:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:0.0:1773941606.526405:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941606.526407:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1773941606.526410:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941606.526412:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941606.526417:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:0.0:1773941606.526418:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1773941606.526422:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10837480. 00000020:00000040:0.0:1773941606.526425:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:0.0:1773941606.526427:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1773941606.526429:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941606.526431:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941606.526433:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941606.526435:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941606.526438:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d02ce5000. 00000100:00000001:0.0:1773941606.526441:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941606.526446:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d10837480 x1860112291497984/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1773941606.526452:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941606.526455:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941606.526458:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000084c3e071] to pc [ptlrpcd_rcv+-1] req@ffff988d10837480 x1860112291497984/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941606.526478:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941606.526481:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:0.0:1773941606.526482:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941606.526484:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941606.526485:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941606.526487:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1773941606.526489:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6840 going to sleep for 1 seconds 00000100:00000001:0.0:1773941606.526490:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941606.526492:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1773941606.526495:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10836a00 x1860112291497856/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941622 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941606.526501:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:0.0:1773941606.526503:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941606.526505:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941606.526507:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1773941606.526510:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10836a00 x1860112291497856/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941622 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941606.526524:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10836a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291497856:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941606.526527:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941606.526529:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941606.526531:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1773941606.526533:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941606.526535:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10836a00 x1860112291497856/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941622 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941606.526540:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1773941606.526542:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb6840. 00000100:00000001:0.0:1773941606.526544:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1773941606.526546:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:0.0:1773941606.526549:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1773941606.526552:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1773941606.526553:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941606.526555:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030400 refcount=4 00000020:00000001:0.0:1773941606.526557:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00010000:00000001:0.0:1773941606.526559:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00010000:00010000:0.0:1773941606.526563:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2abe lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1773941606.526570:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00010000:00010000:0.0:1773941606.526574:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2abe lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1773941606.526580:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1773941606.526582:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2abe lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1773941606.526587:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941606.526595:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:0.0:1773941606.526595:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1773941606.526599:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2abe lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941606.526607:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1773941606.526610:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:3.0:1773941606.526613:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00010000:00000001:0.0:1773941606.526615:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941606.526620:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1773941606.526623:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2abe lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:3.0:1773941606.526632:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10837480 x1860112291497984/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:0.0:1773941606.526636:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d00030400 already unused, calling callback (ffffffffc12b4c20) 10000000:00000001:0.0:1773941606.526644:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1773941606.526650:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2abe lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941606.526677:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00010000:00000001:0.0:1773941606.526680:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000100:00000001:3.0:1773941606.526682:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941606.526686:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1773941606.526688:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000001:3.0:1773941606.526689:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000040:0.0:1773941606.526691:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030400 refcount=5 02000000:00000001:3.0:1773941606.526696:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1773941606.526697:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 02000000:00000001:3.0:1773941606.526712:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941606.526714:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00010000:00000001:0.0:1773941606.526733:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000100:00100000:3.0:1773941606.526740:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10837480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291497984:192.168.204.114@tcp:250:ll_cfg_requeue.0 00010000:00010000:0.0:1773941606.526741:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2abe lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941606.526763:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 10000000:00000001:0.0:1773941606.526769:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 02000000:00000001:3.0:1773941606.526773:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 10000000:00010000:0.0:1773941606.526774:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2abe lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941606.526778:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941606.526787:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 10000000:01000000:0.0:1773941606.526793:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:0.0:1773941606.526798:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1773941606.526800:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941606.526801:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03522400. 00010000:00010000:0.0:1773941606.526807:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 02000000:00000001:3.0:1773941606.526808:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941606.526810:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:3.0:1773941606.526821:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:0.0:1773941606.526822:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1773941606.526830:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000400:00000010:3.0:1773941606.526832:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00010000:00000001:0.0:1773941606.526835:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941606.526842:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1773941606.526847:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d00030400 with handle 0x8e86767622e2abe from hash 00000400:00000010:3.0:1773941606.526848:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00010000:00000001:0.0:1773941606.526853:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1773941606.526857:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941606.526859:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000200:3.0:1773941606.526860:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291497984, portal 25 00010000:00000001:0.0:1773941606.526861:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000100:00000001:3.0:1773941606.526870:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00000001:0.0:1773941606.526872:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1773941606.526876:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1773941606.526880:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941606.526881:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941606.526882:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941606.526883:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941606.526886:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2abe lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1773941606.526891:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941606.526894:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367702144 : -113743341849472 : ffff988d10837480) 00010000:00000001:0.0:1773941606.526895:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941606.526900:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1773941606.526903:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1773941606.526904:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941606.526906:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000040:3.0:1773941606.526907:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10837480 x1860112291497984/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941616 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1773941606.526909:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1773941606.526914:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2abe lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1773941606.526986:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d21ec1400 count: 0 00010000:00000001:0.0:1773941606.526998:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941606.527000:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941606.527003:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941606.527010:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10836a00 x1860112291497856/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941622 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941606.527017:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1773941606.527018:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1773941606.527020:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000040:3.0:1773941606.527025:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:3.0:1773941606.527040:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac3b8. 02000000:00000010:0.0:1773941606.527040:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d02ce6c00. 02000000:00000001:0.0:1773941606.527061:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000200:3.0:1773941606.527062:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291497984, offset 0 00000020:00000001:0.0:1773941606.527070:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941606.527073:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000400:00000200:3.0:1773941606.527086:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000020:00000001:0.0:1773941606.527087:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1773941606.527096:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065c5a00. 02000000:00000001:0.0:1773941606.527102:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000400:00000200:3.0:1773941606.527104:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 02000000:00000001:0.0:1773941606.527106:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1773941606.527114:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10836a00. 00000100:00000001:0.0:1773941606.527119:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000400:00000200:3.0:1773941606.527120:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000100:00000001:0.0:1773941606.527122:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1773941606.527128:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000200:3.0:1773941606.527131:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 10000000:01000000:0.0:1773941606.527136:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:0.0:1773941606.527141:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000400:00000200:3.0:1773941606.527142:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 10000000:00000010:0.0:1773941606.527145:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d10872940. 00000400:00000200:3.0:1773941606.527152:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000020:00000010:0.0:1773941606.527156:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d065c5a00. 00000400:00000200:3.0:1773941606.527161:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941606.527168:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[168, 167] 00000020:00000010:0.0:1773941606.527168:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d065c5e00. 00000040:00000010:0.0:1773941606.527195:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d065c5400. 00000400:00000200:3.0:1773941606.527201:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000020:00000010:0.0:1773941606.527204:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d054a7000. 00000400:00000200:3.0:1773941606.527214:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [279:279:256:4294967295] 192.168.204.124@tcp LPNI seq info [169:169:8:4294967295] 00000020:00000010:0.0:1773941606.527215:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d065c4a00. 80000000:00000010:0.0:1773941606.527226:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d06580a20. 00000400:00000200:3.0:1773941606.527235:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00040000:00000010:0.0:1773941606.527240:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d065c4200. 20000000:00000010:0.0:1773941606.527247:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d065c5c00. 10000000:00000040:0.0:1773941606.527259:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 00000400:00000200:3.0:1773941606.527260:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 10000000:00000001:0.0:1773941606.527271:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000800:00000200:3.0:1773941606.527280:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.124@tcp 00000020:00000010:0.0:1773941606.527281:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d065c5e00. 00000040:00000010:0.0:1773941606.527301:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d065c5400. 00000800:00000010:3.0:1773941606.527302:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74f00. 00000020:00000010:0.0:1773941606.527320:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d054a7000. 00000800:00000200:3.0:1773941606.527326:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000020:00000010:0.0:1773941606.527326:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d065c4a00. 80000000:00000010:0.0:1773941606.527338:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d06580a20. 00000800:00000200:3.0:1773941606.527344:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00040000:00000010:0.0:1773941606.527344:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d065c4200. 00000800:00000200:3.0:1773941606.527355:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74f00 type 1, nob 616 niov 1 nkiov 1 20000000:00000010:0.0:1773941606.527355:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d065c5c00. 00000020:00000010:0.0:1773941606.527368:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d065c5a00. 10000000:00000010:0.0:1773941606.527373:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d10872940. 10000000:01000000:0.0:1773941606.527375:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 00000100:00000001:3.0:1773941606.527381:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941606.527383:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000200:3.0:1773941606.527392:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 10000000:00020000:0.0:1773941606.527394:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 00000100:00000001:3.0:1773941606.527406:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294998123 10000000:00000001:0.0:1773941606.527407:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1773941606.527418:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1773941606.527423:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941606.527426:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000100:00000001:3.0:1773941606.527427:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294998119 00000100:00000001:3.0:1773941606.527440:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294998123 00000100:00000001:3.0:1773941606.527447:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294998119 00000100:00000001:3.0:1773941606.527460:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941606.527463:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941606.527468:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941606.527472:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941606.527480:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941606.527482:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1773941606.527485:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941606.527488:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941606.527489:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941606.527492:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941606.527503:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941606.527514:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74f00. 00000400:00000200:1.0:1773941606.527519:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941606.527526:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941606.527529:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac3b8 00000400:00000010:1.0:1773941606.527531:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac3b8. 00000100:00000001:1.0:1773941606.527534:0:2394:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941606.527539:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10837480 x1860112291497984/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941616 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941606.527546:0:2394:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941606.527549:0:2394:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10837480 x1860112291497984/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941616 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941606.527555:0:2394:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941606.527556:0:2394:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000010:0.1:1773941606.533044:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d21ec1400. 00010000:00000010:0.1:1773941606.533072:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d00030400. 00000800:00000001:1.0:1773941606.533873:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941606.533888:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941606.533891:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941606.533900:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941606.533907:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941606.533914:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:1.0:1773941606.533923:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941606.533928:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941606.533932:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941606.533937:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324608800 00000400:00000200:1.0:1773941606.533942:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941606.533948:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0x11f1 [1] + 0 00000800:00000001:1.0:1773941606.533957:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941606.533993:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941606.533997:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941606.534002:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:1.0:1773941606.534007:0:2393:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941606.534015:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10837480 x1860112291497984/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941616 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941606.534031:0:2393:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d10837480 x1860112291497984/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941616 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941606.534067:0:2393:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1773941606.534077:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941606.534082:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941606.534117:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941606.534126:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941606.534136:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941606.534140:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941606.534145:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941606.534152:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941606.534155:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941606.534160:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941606.534170:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10837480 x1860112291497984/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941616 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941606.534213:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10837480 x1860112291497984/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941616 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941606.534247:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941606.534249:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941606.534252:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941606.534256:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941606.534258:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941606.534261:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941606.534263:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941606.534271:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=107 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941606.534275:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941606.534278:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941606.534280:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941606.534283:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941606.534286:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941606.534289:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941606.534305:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10837480 x1860112291497984/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941616 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941606.534313:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941606.534317:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941606.534319:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941606.534322:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941606.534329:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941606.534332:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941606.534334:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941606.534337:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941606.534340:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941606.534343:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941606.534346:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941606.534350:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941606 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941606.534353:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941606.534361:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941606.534364:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941606.534367:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941606.534370:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941606.534372:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941606.534376:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941606.534378:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941606.534382:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941606.534386:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941606.534389:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941606.534392:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941606.534857:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941606.534865:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941606.534877:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10837480 x1860112291497984/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941616 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941606.534892:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10837480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291497984:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941606.534917:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941606.534921:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941606.534927:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10837480 x1860112291497984/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941616 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941606.534940:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:1.0:1773941606.534940:0:14809:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 02000000:00000001:3.0:1773941606.534955:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941606.535048:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03522400. 00000100:00080000:1.0:1773941606.535051:0:14809:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00100000:1.0:1773941606.535064:0:14809:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000001:3.0:1773941606.535065:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:1.0:1773941606.535070:0:14809:0:(obd_class.h:1215:obd_import_event()) Process entered 00000020:00000001:3.0:1773941606.535073:0:2400:0:(genops.c:1236:class_import_put()) Process entered 10000000:01000000:1.0:1773941606.535075:0:14809:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000020:00000040:3.0:1773941606.535081:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000001:00000001:1.0:1773941606.535082:0:14809:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941606.535087:0:14809:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000001:3.0:1773941606.535098:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:1.0:1773941606.535098:0:14809:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941606.535123:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d02ce5000. 00000001:00000001:1.0:1773941606.535125:0:14809:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941606.535129:0:14809:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941606.535138:0:14809:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941606.535140:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 10000000:00000001:1.0:1773941606.535143:0:14809:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941606.535147:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1773941606.535148:0:14809:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000010:3.0:1773941606.535159:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10837480. 00000100:00080000:1.0:1773941606.535162:0:14809:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:1.0:1773941606.535200:0:14809:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:3.0:1773941606.535204:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941606.535205:0:14809:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941606.535212:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941606.535214:0:14809:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941606.535248:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1773941606.535248:0:14809:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941606.535260:0:14809:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:3.0:1773941606.535261:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 10000000:01000000:1.0:1773941606.535263:0:14809:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000100:00000001:3.0:1773941606.535265:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 10000000:00000040:1.0:1773941606.535266:0:14809:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00000001:3.0:1773941606.535274:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941606.535277:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941606.535291:0:14809:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941606.535296:0:14809:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941606.535299:0:14809:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941606.535305:0:14809:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:1.0:1773941606.535308:0:14809:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941606.535311:0:14809:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941606.535313:0:14809:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941606.535316:0:14809:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941606.535317:0:14809:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941607.419246:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941607.419256:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941607.419262:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:0.0:1773941607.547117:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1773941607.547129:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941607.547139:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941607.547145:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941607.547148:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941607.547150:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1773941607.547177:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:0.0:1773941607.547181:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:0.0:1773941607.547186:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:0.0:1773941607.547192:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1773941607.547196:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:0.0:1773941607.547213:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10836a00. 00000020:00000040:0.0:1773941607.547223:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941607.547233:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941607.547239:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941607.547247:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941607.547260:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065c5a00. 00000100:00000001:0.0:1773941607.547274:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941607.547282:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1773941607.547288:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1773941607.547300:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d21ec1400. 00010000:00000001:0.0:1773941607.547309:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1773941607.547316:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d00030400. 00000020:00000001:0.0:1773941607.547323:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:0.0:1773941607.547328:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d00030400 with handle 0x8e86767622e2ac5 to hash 00000020:00000001:0.0:1773941607.547333:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:0.0:1773941607.547337:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00010000:00000001:0.0:1773941607.547343:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00010000:00000001:0.0:1773941607.547350:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941607.547376:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941607.547386:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ac5 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941607.547409:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ac5 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941607.547432:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ac5 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941607.547445:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1773941607.547449:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941607.547461:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb6840. 00000100:00000001:0.0:1773941607.547466:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330139633728 : -113743569917888 : ffff988d02eb6840) 00000100:00000001:0.0:1773941607.547472:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941607.547475:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367699456 : -113743341852160 : ffff988d10836a00) 00000100:00000001:0.0:1773941607.547483:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941607.547492:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941607.547497:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941607.547501:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941607.547519:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10836a00 x1860112291498112/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941607.547539:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941607.547543:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941607.547549:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941607.547555:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941607.547560:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941607.547568:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10836a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291498112:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941607.547577:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941607.547585:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941607.547589:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941607.547594:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941607.547602:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d02ce6c00. 02000000:00000001:0.0:1773941607.547619:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941607.547633:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941607.547641:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0014baa8. 00000400:00000010:0.0:1773941607.547654:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d108336e8. 00000100:00000200:0.0:1773941607.547666:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291498112, portal 25 00000100:00000001:0.0:1773941607.547671:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941607.547673:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367699456 : -113743341852160 : ffff988d10836a00) 00000100:00000040:0.0:1773941607.547686:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10836a00 x1860112291498112/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941623 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941607.547702:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941607.547708:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941607.547714:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833088. 00000100:00000200:0.0:1773941607.547719:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291498112, offset 0 00000400:00000200:0.0:1773941607.547731:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941607.547745:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941607.547757:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941607.547766:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941607.547774:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941607.547781:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941607.547787:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941607.547794:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[168, 169] 00000400:00000200:0.0:1773941607.547802:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:0.0:1773941607.547810:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [280:280:256:4294967295] 192.168.204.114@tcp LPNI seq info [170:170:8:4294967295] 00000400:00000200:0.0:1773941607.547819:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941607.547836:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:0.0:1773941607.547848:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:0.0:1773941607.547857:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d113e9d00. 00000800:00000200:0.0:1773941607.547870:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:0.0:1773941607.547883:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:0.0:1773941607.547891:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d113e9d00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1773941607.547926:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941607.547950:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941607.547956:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294999143 00000100:00000001:0.0:1773941607.548010:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294998123 00000100:00000001:0.0:1773941607.548018:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294999144 00000100:00000001:0.0:1773941607.548026:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294998123 00000100:00000001:0.0:1773941607.548032:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941607.548036:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941607.548040:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941607.548043:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1773941607.548047:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6840 going to sleep for 11 seconds 00000100:00000001:0.0:1773941607.548052:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941607.548057:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941607.548195:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941607.548216:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d113e9d00. 00000400:00000200:1.0:1773941607.548224:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941607.548232:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941607.548240:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833088 00000400:00000010:1.0:1773941607.548243:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833088. 00000100:00000001:1.0:1773941607.548249:0:2394:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941607.548257:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10836a00 x1860112291498112/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941623 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941607.548273:0:2394:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941607.548278:0:2394:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d10836a00 x1860112291498112/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941623 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941607.548288:0:2394:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941607.548290:0:2394:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:1.0:1773941607.549568:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941607.549589:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941607.549593:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941607.549597:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941607.549604:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941607.549612:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941607.549620:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941607.549625:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941607.549631:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941607.549637:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324608880 00000400:00000200:1.0:1773941607.549643:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941607.549649:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0x11f9 [1] + 224 00000800:00000001:1.0:1773941607.549658:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941607.549669:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941607.549672:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941607.549677:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:1.0:1773941607.549684:0:2393:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941607.549691:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10836a00 x1860112291498112/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941623 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941607.549706:0:2393:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d10836a00 x1860112291498112/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941623 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941607.549746:0:2393:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1773941607.549758:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941607.549763:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941607.549829:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941607.549842:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941607.549846:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941607.549851:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0014baa8. 00000400:00000200:0.0:1773941607.549860:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d108336e8 00000400:00000010:0.0:1773941607.549864:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d108336e8. 00000100:00000001:0.0:1773941607.549870:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941607.549882:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10836a00 x1860112291498112/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941623 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941607.549905:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10836a00 x1860112291498112/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941623 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941607.549921:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941607.549925:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941607.549931:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941607.549936:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941607.549941:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941607.549945:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941607.549952:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941607.550000:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=108 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941607.550009:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941607.550015:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:0.0:1773941607.550024:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:0.0:1773941607.550043:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941607.550050:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:0.0:1773941607.550057:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:0.0:1773941607.550064:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:0.0:1773941607.550074:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:0.0:1773941607.550081:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:0.0:1773941607.550085:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:0.0:1773941607.550089:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:0.0:1773941607.550093:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:0.0:1773941607.550102:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d10836a00 x1860112291498112/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941623 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941607.550118:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:0.0:1773941607.550121:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:0.0:1773941607.550125:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941607.550130:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:0.0:1773941607.550134:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:0.0:1773941607.550138:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941607.550142:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941607.550145:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941607.550147:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:0.0:1773941607.550151:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941607.550183:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941607.550188:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1773941607.550192:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1773941607.550199:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1773941607.550205:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 329 00000400:00000200:0.0:1773941607.550215:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941607.550224:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:0.0:1773941607.550233:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941607.550239:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941607.550244:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941607.550249:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:0.0:1773941607.550268:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941607.550271:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:0.0:1773941607.550276:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941607.550281:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:0.0:1773941607.550286:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:0.0:1773941607.550291:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:0.0:1773941607.550299:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941607.550303:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941607.550307:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941607.550311:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:0.0:1773941607.550317:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:0.0:1773941607.550324:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:0.0:1773941607.550330:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941607.550335:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1773941607.550341:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941607.550346:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941607.550351:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:0.0:1773941607.550355:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1773941607.550363:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10836300. 00000020:00000040:0.0:1773941607.550368:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:0.0:1773941607.550374:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1773941607.550378:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941607.550383:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941607.550387:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941607.550391:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941607.550401:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d02ce5000. 00000100:00000001:0.0:1773941607.550410:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941607.550422:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d10836300 x1860112291498240/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1773941607.550439:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941607.550444:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941607.550454:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000831309b7] to pc [ptlrpcd_rcv+-1] req@ffff988d10836300 x1860112291498240/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941607.550516:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941607.550523:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:0.0:1773941607.550527:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941607.550533:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941607.550538:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941607.550540:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1773941607.550544:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6840 going to sleep for 1 seconds 00000100:00000001:0.0:1773941607.550549:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941607.550553:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1773941607.550566:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10836a00 x1860112291498112/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941623 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941607.550583:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941607.550585:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941607.550593:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941607.550595:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941607.550599:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1773941607.550604:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941607.550611:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941607.550615:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10836300 x1860112291498240/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941607.550629:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10836a00 x1860112291498112/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941623 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941607.550639:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941607.550645:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941607.550654:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941607.550659:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941607.550667:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941607.550673:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10836a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291498112:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00100000:3.0:1773941607.550676:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10836300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291498240:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941607.550694:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1773941607.550696:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941607.550704:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941607.550714:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 02000000:00000001:3.0:1773941607.550715:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1773941607.550719:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1773941607.550721:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941607.550727:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941607.550733:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03522400. 00000100:00000040:0.0:1773941607.550736:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10836a00 x1860112291498112/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941623 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1773941607.550739:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941607.550756:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941607.550765:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00000100:00000001:0.0:1773941607.550774:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000010:3.0:1773941607.550777:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00000100:00000010:0.0:1773941607.550786:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb6840. 00000100:00000200:3.0:1773941607.550787:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291498240, portal 25 00000100:00000001:3.0:1773941607.550797:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941607.550801:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:3.0:1773941607.550802:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367697664 : -113743341853952 : ffff988d10836300) 00000100:00000001:0.0:1773941607.550811:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:3.0:1773941607.550818:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10836300 x1860112291498240/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941617 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1773941607.550823:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000001:3.0:1773941607.550832:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941607.550836:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:3.0:1773941607.550847:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acf68. 00010000:00000001:0.0:1773941607.550848:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000200:3.0:1773941607.550851:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291498240, offset 0 00000020:00000001:0.0:1773941607.550856:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:3.0:1773941607.550865:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000020:00000040:0.0:1773941607.550866:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030400 refcount=4 00000400:00000200:3.0:1773941607.550876:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000020:00000001:0.0:1773941607.550878:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00000400:00000200:3.0:1773941607.550888:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:0.0:1773941607.550889:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00000400:00000200:3.0:1773941607.550896:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941607.550905:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00010000:0.0:1773941607.550906:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ac5 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941607.550915:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941607.550925:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941607.550935:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[170, 169] 00010000:00000001:0.0:1773941607.550944:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000200:3.0:1773941607.550947:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:3.0:1773941607.550958:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [281:281:256:4294967295] 192.168.204.124@tcp LPNI seq info [171:171:8:4294967295] 00010000:00010000:0.0:1773941607.551015:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ac5 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941607.551016:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:3.0:1773941607.551041:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00010000:00000001:0.0:1773941607.551052:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000800:00000200:3.0:1773941607.551055:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.124@tcp 00010000:00010000:0.0:1773941607.551064:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ac5 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000010:3.0:1773941607.551065:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74c00. 00000800:00000200:3.0:1773941607.551080:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00010000:00000001:0.0:1773941607.551092:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941607.551099:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000200:3.0:1773941607.551107:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00010000:00010000:0.0:1773941607.551109:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ac5 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941607.551124:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74c00 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:0.0:1773941607.551128:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941607.551135:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941607.551142:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000100:00000001:3.0:1773941607.551145:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941607.551149:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ac5 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941607.551185:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:3.0:1773941607.551195:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4294999147 00010000:00010000:0.0:1773941607.551206:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d00030400 already unused, calling callback (ffffffffc12b4c20) 00000100:00000001:3.0:1773941607.551208:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4294999143 10000000:00000001:0.0:1773941607.551217:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000001:3.0:1773941607.551220:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4294999147 10000000:00010000:0.0:1773941607.551229:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ac5 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941607.551230:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4294999144 00000100:00000001:3.0:1773941607.551243:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941607.551252:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941607.551258:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000100:00000001:3.0:1773941607.551259:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941607.551264:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941607.551273:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941607.551278:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00010000:00000001:0.0:1773941607.551278:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000800:00000200:1.0:1773941607.551286:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000020:00000001:0.0:1773941607.551286:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:3.0:1773941607.551287:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941607.551296:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000800:00000010:1.0:1773941607.551299:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74c00. 00000020:00000040:0.0:1773941607.551299:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030400 refcount=5 00000100:00000001:3.0:1773941607.551302:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941607.551312:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00000400:00000200:1.0:1773941607.551313:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:3.0:1773941607.551315:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941607.551328:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00010000:00000001:0.0:1773941607.551338:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:1.0:1773941607.551339:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00010000:00010000:0.0:1773941607.551350:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ac5 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:1.0:1773941607.551351:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acf68 00000400:00000010:1.0:1773941607.551359:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acf68. 00000100:00000001:1.0:1773941607.551369:0:2394:0:(events.c:34:request_out_callback()) Process entered 10000000:00000001:0.0:1773941607.551379:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000200:1.0:1773941607.551382:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10836300 x1860112291498240/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941617 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00010000:0.0:1773941607.551388:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ac5 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941607.551413:0:2394:0:(client.c:2842:__ptlrpc_req_put()) Process entered 10000000:01000000:0.0:1773941607.551423:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000100:00000040:1.0:1773941607.551424:0:2394:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10836300 x1860112291498240/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941617 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000040:0.0:1773941607.551439:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1773941607.551445:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941607.551449:0:2394:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941607.551455:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000100:00000001:1.0:1773941607.551456:0:2394:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:0.0:1773941607.551465:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1773941607.551472:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1773941607.551476:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1773941607.551482:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941607.551487:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1773941607.551495:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d00030400 with handle 0x8e86767622e2ac5 from hash 00010000:00000001:0.0:1773941607.551501:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1773941607.551504:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941607.551506:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941607.551509:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1773941607.551512:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1773941607.551515:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1773941607.551521:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941607.551524:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941607.551527:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941607.551531:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941607.551537:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ac5 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1773941607.551571:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941607.551575:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941607.551578:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1773941607.551581:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1773941607.551585:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941607.551587:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941607.551590:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1773941607.551595:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2ac5 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1773941607.551609:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d21ec1400 count: 0 00010000:00000001:0.0:1773941607.551627:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941607.551631:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941607.551636:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941607.551648:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10836a00 x1860112291498112/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941623 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941607.551664:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1773941607.551669:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1773941607.551674:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d02ce6c00. 02000000:00000001:0.0:1773941607.551682:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1773941607.551686:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941607.551689:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:0.0:1773941607.551694:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1773941607.551699:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065c5a00. 02000000:00000001:0.0:1773941607.551706:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1773941607.551709:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1773941607.551713:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10836a00. 00000100:00000001:0.0:1773941607.551719:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941607.551722:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1773941607.551726:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:0.0:1773941607.551731:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:0.0:1773941607.551736:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1773941607.551744:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d10872940. 00000020:00000010:0.0:1773941607.551752:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d065c5a00. 00000020:00000010:0.0:1773941607.551777:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d065c5c00. 00000040:00000010:0.0:1773941607.551785:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d065c4200. 00000020:00000010:0.0:1773941607.551794:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d054a7000. 00000020:00000010:0.0:1773941607.551803:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d065c4a00. 80000000:00000010:0.0:1773941607.551810:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d06580a20. 00040000:00000010:0.0:1773941607.551818:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d065c5400. 20000000:00000010:0.0:1773941607.551826:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d065c5e00. 10000000:00000040:0.0:1773941607.551834:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:0.0:1773941607.551839:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:0.0:1773941607.551848:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d065c5c00. 00000040:00000010:0.0:1773941607.551856:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d065c4200. 00000020:00000010:0.0:1773941607.551863:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d054a7000. 00000020:00000010:0.0:1773941607.551870:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d065c4a00. 80000000:00000010:0.0:1773941607.551876:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d06580a20. 00040000:00000010:0.0:1773941607.551882:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d065c5400. 20000000:00000010:0.0:1773941607.551888:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d065c5e00. 00000020:00000010:0.0:1773941607.551893:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d065c5a00. 10000000:00000010:0.0:1773941607.551898:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d10872940. 10000000:01000000:0.0:1773941607.551904:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:0.0:1773941607.551909:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:0.0:1773941607.551914:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:0.0:1773941607.551918:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1773941607.551921:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1773941607.551924:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941607.551929:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:1.0:1773941607.553747:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941607.553760:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941607.553764:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941607.553767:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941607.553773:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941607.553780:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:1.0:1773941607.553789:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941607.553793:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941607.553798:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941607.553803:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324608900 00000400:00000200:1.0:1773941607.553808:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941607.553814:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0x1209 [1] + 0 00000800:00000001:1.0:1773941607.553822:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941607.553833:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941607.553837:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941607.553853:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:1.0:1773941607.553859:0:2393:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941607.553866:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10836300 x1860112291498240/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941617 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941607.553881:0:2393:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d10836300 x1860112291498240/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941617 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941607.553908:0:2393:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1773941607.553917:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941607.553922:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941607.553982:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941607.553988:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941607.553995:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941607.553997:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941607.554001:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941607.554008:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941607.554013:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941607.554018:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941607.554029:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10836300 x1860112291498240/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941617 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941607.554045:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10836300 x1860112291498240/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941617 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941607.554056:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941607.554058:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941607.554061:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941607.554075:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941607.554077:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941607.554080:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941607.554083:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941607.554088:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=108 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941607.554092:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941607.554096:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941607.554098:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941607.554101:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941607.554104:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941607.554107:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941607.554113:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10836300 x1860112291498240/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941617 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941607.554122:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941607.554125:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941607.554128:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941607.554131:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941607.554137:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941607.554139:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941607.554141:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941607.554143:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941607.554146:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941607.554149:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941607.554152:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941607.554180:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941607 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941607.554184:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941607.554193:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941607.554196:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941607.554198:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941607.554201:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941607.554203:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941607.554208:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941607.554211:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941607.554214:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941607.554227:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941607.554232:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941607.554235:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941607.554712:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941607.554719:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941607.554730:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10836300 x1860112291498240/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941617 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941607.554747:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10836300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291498240:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941607.554759:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941607.554762:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1773941607.554769:0:14810:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000040:3.0:1773941607.554772:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10836300 x1860112291498240/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941617 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941607.554787:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941607.554790:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00080000:1.0:1773941607.554791:0:14810:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 02000000:00000010:3.0:1773941607.554795:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03522400. 00000100:00100000:1.0:1773941607.554800:0:14810:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000001:3.0:1773941607.554803:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:1.0:1773941607.554809:0:14810:0:(obd_class.h:1215:obd_import_event()) Process entered 00000020:00000001:3.0:1773941607.554810:0:2400:0:(genops.c:1236:class_import_put()) Process entered 10000000:01000000:1.0:1773941607.554816:0:14810:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000020:00000040:3.0:1773941607.554817:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000001:00000001:1.0:1773941607.554826:0:14810:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:3.0:1773941607.554827:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:1.0:1773941607.554837:0:14810:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000010:3.0:1773941607.554838:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d02ce5000. 02000000:00000001:3.0:1773941607.554849:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000001:00000001:1.0:1773941607.554850:0:14810:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941607.554852:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000001:00000001:1.0:1773941607.554857:0:14810:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000010:3.0:1773941607.554859:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10836300. 00000001:00000001:1.0:1773941607.554863:0:14810:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:3.0:1773941607.554867:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000001:00000001:1.0:1773941607.554869:0:14810:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941607.554872:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941607.554874:0:14810:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941607.554879:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941607.554881:0:14810:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941607.554887:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:1.0:1773941607.554891:0:14810:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:3.0:1773941607.554893:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941607.554900:0:14810:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:3.0:1773941607.554902:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941607.554907:0:14810:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941607.554908:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941607.554916:0:14810:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941607.554921:0:14810:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941607.554924:0:14810:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941607.554926:0:14810:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941607.554929:0:14810:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:1.0:1773941607.554952:0:14810:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941607.554955:0:14810:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941607.554957:0:14810:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941607.555002:0:14810:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:1.0:1773941607.555025:0:14810:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941607.555029:0:14810:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941607.555032:0:14810:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941607.555036:0:14810:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941607.555039:0:14810:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:0.1:1773941607.557015:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d21ec1400. 00010000:00000010:0.1:1773941607.557029:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d00030400. 00000100:00000040:2.0:1773941607.675110:0:461:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (335) 00000020:00000001:1.0:1773941608.443182:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1773941608.443188:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941608.443192:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1773941608.827093:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4294999147, at 4294999168 00000100:00000040:1.0:1773941608.827098:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4294999147, at 4294999168 00000100:00000010:0.0:1773941608.827110:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580b40. 00000100:00000010:1.0:1773941608.827114:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580d80. 10000000:00000001:0.0:1773941609.339056:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1773941609.339064:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941609.339068:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941609.339071:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941609.339072:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941609.339074:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1773941609.339076:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:0.0:1773941609.339096:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:0.0:1773941609.339099:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:0.0:1773941609.339102:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1773941609.339103:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:0.0:1773941609.339112:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10836a00. 00000020:00000040:0.0:1773941609.339117:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941609.339152:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941609.339156:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941609.339162:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941609.339170:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065c5a00. 00000100:00000001:0.0:1773941609.339177:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941609.339181:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1773941609.339184:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1773941609.339191:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d21ec1400. 00010000:00000001:0.0:1773941609.339197:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1773941609.339200:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d00030400. 00000020:00000001:0.0:1773941609.339204:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:0.0:1773941609.339207:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d00030400 with handle 0x8e86767622e2acc to hash 00000020:00000001:0.0:1773941609.339209:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:0.0:1773941609.339211:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00010000:00000001:0.0:1773941609.339213:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00010000:00000001:0.0:1773941609.339217:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941609.339220:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941609.339229:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2acc lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941609.339240:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2acc lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941609.339251:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2acc lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941609.339257:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1773941609.339259:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941609.339270:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb6840. 00000100:00000001:0.0:1773941609.339272:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330139633728 : -113743569917888 : ffff988d02eb6840) 00000100:00000001:0.0:1773941609.339275:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941609.339277:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367699456 : -113743341852160 : ffff988d10836a00) 00000100:00000001:0.0:1773941609.339281:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941609.339285:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941609.339288:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941609.339290:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941609.339300:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10836a00 x1860112291498368/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941609.339310:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941609.339312:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941609.339315:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941609.339318:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941609.339320:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941609.339324:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10836a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291498368:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941609.339329:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941609.339334:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941609.339336:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941609.339338:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941609.339342:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d02ce6c00. 02000000:00000001:0.0:1773941609.339344:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941609.339351:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941609.339355:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0014baa8. 00000400:00000010:0.0:1773941609.339360:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d108336e8. 00000100:00000200:0.0:1773941609.339367:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291498368, portal 25 00000100:00000001:0.0:1773941609.339369:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941609.339370:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367699456 : -113743341852160 : ffff988d10836a00) 00000100:00000040:0.0:1773941609.339375:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10836a00 x1860112291498368/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941625 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941609.339383:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941609.339385:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941609.339387:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833bb0. 00000100:00000200:0.0:1773941609.339395:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291498368, offset 0 00000400:00000200:0.0:1773941609.339401:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941609.339409:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941609.339416:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941609.339422:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941609.339426:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941609.339431:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941609.339434:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941609.339437:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[170, 171] 00000400:00000200:0.0:1773941609.339441:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:0.0:1773941609.339445:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [282:282:256:4294967295] 192.168.204.114@tcp LPNI seq info [172:172:8:4294967295] 00000400:00000200:0.0:1773941609.339449:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941609.339458:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:0.0:1773941609.339463:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:0.0:1773941609.339468:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d113e9400. 00000800:00000200:0.0:1773941609.339475:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:0.0:1773941609.339483:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:0.0:1773941609.339487:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d113e9400 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1773941609.339506:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941609.339510:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941609.339513:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4295000935 00000100:00000010:0.0:1773941609.339516:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580b40. 00000100:00000001:0.0:1773941609.339523:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1773941609.339530:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4295000935 00000100:00000010:0.0:1773941609.339532:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580a20. 00000100:00000001:0.0:1773941609.339535:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1773941609.339539:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941609.339542:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941609.339545:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941609.339547:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1773941609.339556:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6840 going to sleep for 11 seconds 00000100:00000001:0.0:1773941609.339559:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941609.339562:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941609.339691:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941609.339700:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d113e9400. 00000400:00000200:1.0:1773941609.339709:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941609.339717:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941609.339724:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833bb0 00000400:00000010:1.0:1773941609.339727:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833bb0. 00000100:00000001:1.0:1773941609.339733:0:2394:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941609.339740:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10836a00 x1860112291498368/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941625 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941609.339755:0:2394:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941609.339761:0:2394:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d10836a00 x1860112291498368/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941625 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941609.339772:0:2394:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941609.339775:0:2394:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:1.0:1773941609.341071:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941609.341089:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941609.341093:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941609.341097:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941609.341106:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941609.341114:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941609.341138:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941609.341143:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941609.341149:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941609.341163:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324608980 00000400:00000200:1.0:1773941609.341169:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941609.341176:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0x1211 [1] + 224 00000800:00000001:1.0:1773941609.341185:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941609.341197:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941609.341201:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941609.341206:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:1.0:1773941609.341213:0:2393:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941609.341220:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10836a00 x1860112291498368/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941625 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941609.341234:0:2393:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d10836a00 x1860112291498368/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941625 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941609.341269:0:2393:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1773941609.341280:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941609.341287:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941609.341300:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941609.341310:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941609.341313:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941609.341317:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0014baa8. 00000400:00000200:0.0:1773941609.341324:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d108336e8 00000400:00000010:0.0:1773941609.341326:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d108336e8. 00000100:00000001:0.0:1773941609.341330:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941609.341338:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10836a00 x1860112291498368/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941625 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941609.341355:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10836a00 x1860112291498368/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941625 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941609.341365:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941609.341368:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941609.341370:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941609.341373:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941609.341376:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941609.341378:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941609.341381:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941609.341387:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=110 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941609.341391:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941609.341393:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:0.0:1773941609.341397:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:0.0:1773941609.341400:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941609.341404:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:0.0:1773941609.341407:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:0.0:1773941609.341411:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:0.0:1773941609.341415:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:0.0:1773941609.341419:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:0.0:1773941609.341421:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:0.0:1773941609.341431:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:0.0:1773941609.341434:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:0.0:1773941609.341438:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d10836a00 x1860112291498368/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941625 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941609.341446:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:0.0:1773941609.341447:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:0.0:1773941609.341449:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941609.341452:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:0.0:1773941609.341454:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:0.0:1773941609.341456:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941609.341457:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941609.341459:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941609.341460:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:0.0:1773941609.341462:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941609.341463:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941609.341465:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1773941609.341467:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1773941609.341469:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1773941609.341472:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 330 00000400:00000200:0.0:1773941609.341476:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941609.341480:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:0.0:1773941609.341483:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941609.341486:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941609.341489:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941609.341490:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:0.0:1773941609.341493:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941609.341495:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:0.0:1773941609.341497:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941609.341500:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:0.0:1773941609.341502:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:0.0:1773941609.341505:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:0.0:1773941609.341509:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941609.341512:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941609.341515:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941609.341523:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:0.0:1773941609.341527:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:0.0:1773941609.341532:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:0.0:1773941609.341536:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941609.341539:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1773941609.341543:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941609.341547:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941609.341551:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:0.0:1773941609.341554:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1773941609.341559:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10835180. 00000020:00000040:0.0:1773941609.341563:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:0.0:1773941609.341567:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1773941609.341570:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941609.341573:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941609.341576:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941609.341578:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941609.341584:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d02ce4800. 00000100:00000001:0.0:1773941609.341590:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941609.341598:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d10835180 x1860112291498496/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1773941609.341610:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941609.341614:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941609.341620:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000052b48ae6] to pc [ptlrpcd_rcv+-1] req@ffff988d10835180 x1860112291498496/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941609.341649:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941609.341652:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:0.0:1773941609.341654:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941609.341657:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941609.341660:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941609.341661:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1773941609.341663:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6840 going to sleep for 1 seconds 00000100:00000001:0.0:1773941609.341665:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941609.341667:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1773941609.341681:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10836a00 x1860112291498368/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941625 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941609.341695:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:0.0:1773941609.341698:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941609.341700:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941609.341702:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1773941609.341706:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10836a00 x1860112291498368/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941625 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941609.341714:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10836a00 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291498368:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941609.341719:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941609.341721:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941609.341724:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1773941609.341726:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941609.341729:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10836a00 x1860112291498368/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941625 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941609.341735:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941609.341736:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941609.341741:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000010:0.0:1773941609.341742:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb6840. 00000100:00000001:3.0:1773941609.341746:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1773941609.341748:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1773941609.341750:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:0.0:1773941609.341753:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1773941609.341755:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1773941609.341760:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941609.341762:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030400 refcount=4 00000020:00000001:0.0:1773941609.341766:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00000100:00000040:3.0:1773941609.341768:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10835180 x1860112291498496/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1773941609.341770:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00010000:00010000:0.0:1773941609.341779:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2acc lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941609.341786:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941609.341789:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941609.341794:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00010000:00000001:0.0:1773941609.341794:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 02000000:00000001:3.0:1773941609.341798:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00010000:00010000:0.0:1773941609.341802:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2acc lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941609.341803:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941609.341807:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10835180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291498496:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941609.341811:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941609.341815:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941609.341818:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941609.341823:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:0.0:1773941609.341826:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 02000000:00000010:3.0:1773941609.341829:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03522400. 00010000:00010000:0.0:1773941609.341832:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2acc lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941609.341833:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941609.341845:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:0.0:1773941609.341849:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000010:3.0:1773941609.341851:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00010000:00000001:0.0:1773941609.341853:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000010:3.0:1773941609.341859:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00010000:00010000:0.0:1773941609.341859:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2acc lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:3.0:1773941609.341867:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291498496, portal 25 00010000:00000001:0.0:1773941609.341869:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:3.0:1773941609.341871:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00000001:0.0:1773941609.341873:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941609.341875:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367693184 : -113743341858432 : ffff988d10835180) 00010000:00000001:0.0:1773941609.341878:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1773941609.341883:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2acc lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:3.0:1773941609.341884:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10835180 x1860112291498496/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941619 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:0.0:1773941609.341905:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d00030400 already unused, calling callback (ffffffffc12b4c20) 00000100:00000001:3.0:1773941609.341908:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 10000000:00000001:0.0:1773941609.341910:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000040:3.0:1773941609.341914:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 10000000:00010000:0.0:1773941609.341916:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2acc lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:3.0:1773941609.341918:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac6e8. 00000100:00000200:3.0:1773941609.341925:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291498496, offset 0 00010000:00000001:0.0:1773941609.341928:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1773941609.341931:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:3.0:1773941609.341936:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000020:00000001:0.0:1773941609.341936:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941609.341941:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00030400 refcount=5 00000400:00000200:3.0:1773941609.341947:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000020:00000001:0.0:1773941609.341948:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00010000:00000001:0.0:1773941609.341953:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090849280 : -113743618702336 : ffff988d00030400) 00000400:00000200:3.0:1773941609.341957:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941609.341990:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:0.0:1773941609.341990:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1773941609.341996:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2acc lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941609.341997:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941609.342004:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941609.342011:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 10000000:00000001:0.0:1773941609.342012:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:3.0:1773941609.342024:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[172, 171] 10000000:00010000:0.0:1773941609.342026:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2acc lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941609.342032:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 10000000:01000000:0.0:1773941609.342040:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:3.0:1773941609.342041:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [283:283:256:4294967295] 192.168.204.124@tcp LPNI seq info [173:173:8:4294967295] 10000000:00000040:0.0:1773941609.342051:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000400:00000200:3.0:1773941609.342054:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 10000000:00000001:0.0:1773941609.342056:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941609.342062:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1773941609.342068:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:3.0:1773941609.342071:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00010000:00000001:0.0:1773941609.342073:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1773941609.342078:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1773941609.342082:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000800:00000200:3.0:1773941609.342083:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.124@tcp 00010000:00000001:0.0:1773941609.342085:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000010:3.0:1773941609.342089:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74800. 00000020:00000040:0.0:1773941609.342090:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d00030400 with handle 0x8e86767622e2acc from hash 00010000:00000001:0.0:1773941609.342097:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1773941609.342098:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941609.342101:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000200:3.0:1773941609.342102:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00010000:00000001:0.0:1773941609.342104:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1773941609.342110:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1773941609.342112:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000800:00000200:3.0:1773941609.342117:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00010000:00000001:0.0:1773941609.342118:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941609.342138:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941609.342139:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941609.342143:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74800 type 1, nob 616 niov 1 nkiov 1 10000000:00000001:0.0:1773941609.342143:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941609.342151:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2acc lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1773941609.342157:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941609.342158:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941609.342160:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000100:00000001:3.0:1773941609.342162:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941609.342163:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1773941609.342166:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941609.342169:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:3.0:1773941609.342171:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:0.0:1773941609.342172:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941609.342178:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4295000938 00010000:00010000:0.0:1773941609.342178:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d00030400/0x8e86767622e2acc lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941609.342187:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4295000935 00010000:00000040:0.0:1773941609.342189:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d21ec1400 count: 0 00000100:00000001:3.0:1773941609.342194:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4295000938 00000100:00000001:3.0:1773941609.342203:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4295000935 00000100:00000001:3.0:1773941609.342209:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941609.342212:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941609.342214:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:3.0:1773941609.342216:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941609.342219:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941609.342221:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941609.342226:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1773941609.342231:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000040:0.0:1773941609.342235:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10836a00 x1860112291498368/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941625 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941609.342236:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1773941609.342242:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941609.342245:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941609.342249:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941609.342256:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941609.342257:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1773941609.342263:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1773941609.342268:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d02ce6c00. 00000800:00000200:1.0:1773941609.342269:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 02000000:00000001:0.0:1773941609.342274:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000800:00000010:1.0:1773941609.342277:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74800. 00000020:00000001:0.0:1773941609.342279:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941609.342286:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000400:00000200:1.0:1773941609.342287:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000001:0.0:1773941609.342291:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1773941609.342297:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065c5a00. 00000400:00000200:1.0:1773941609.342299:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 02000000:00000001:0.0:1773941609.342307:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000400:00000200:1.0:1773941609.342309:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac6e8 02000000:00000001:0.0:1773941609.342313:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000400:00000010:1.0:1773941609.342315:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac6e8. 00000100:00000010:0.0:1773941609.342318:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10836a00. 00000100:00000001:1.0:1773941609.342323:0:2394:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1773941609.342326:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941609.342332:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:1.0:1773941609.342334:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10835180 x1860112291498496/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941619 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000001:0.0:1773941609.342338:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:0.0:1773941609.342345:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:0.0:1773941609.342350:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1773941609.342357:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d10872940. 00000100:00000001:1.0:1773941609.342360:0:2394:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000020:00000010:0.0:1773941609.342363:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d065c5a00. 00000100:00000040:1.0:1773941609.342370:0:2394:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10835180 x1860112291498496/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941619 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:0.0:1773941609.342371:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d065c5e00. 00000040:00000010:0.0:1773941609.342377:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d065c5400. 00000020:00000010:0.0:1773941609.342384:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d054a7000. 00000020:00000010:0.0:1773941609.342392:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d065c4a00. 00000100:00000001:1.0:1773941609.342393:0:2394:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 80000000:00000010:0.0:1773941609.342399:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d06580420. 00000100:00000001:1.0:1773941609.342401:0:2394:0:(events.c:68:request_out_callback()) Process leaving 00040000:00000010:0.0:1773941609.342406:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d065c4200. 20000000:00000010:0.0:1773941609.342411:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d065c5c00. 10000000:00000040:0.0:1773941609.342415:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:0.0:1773941609.342416:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:0.0:1773941609.342421:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d065c5e00. 00000040:00000010:0.0:1773941609.342426:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d065c5400. 00000020:00000010:0.0:1773941609.342435:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d054a7000. 00000020:00000010:0.0:1773941609.342439:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d065c4a00. 80000000:00000010:0.0:1773941609.342443:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d06580420. 00040000:00000010:0.0:1773941609.342446:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d065c4200. 20000000:00000010:0.0:1773941609.342449:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d065c5c00. 00000020:00000010:0.0:1773941609.342451:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d065c5a00. 10000000:00000010:0.0:1773941609.342453:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d10872940. 10000000:01000000:0.0:1773941609.342456:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:0.0:1773941609.342459:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:0.0:1773941609.342460:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:0.0:1773941609.342462:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1773941609.342464:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1773941609.342466:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941609.342469:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00010000:00000010:0.1:1773941609.348047:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d21ec1400. 00010000:00000010:0.1:1773941609.348054:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d00030400. 00000800:00000001:1.0:1773941609.348419:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941609.348427:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941609.348430:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941609.348440:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941609.348445:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941609.348449:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:1.0:1773941609.348455:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941609.348459:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941609.348463:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941609.348467:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324608a00 00000400:00000200:1.0:1773941609.348490:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941609.348496:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0x1221 [1] + 0 00000800:00000001:1.0:1773941609.348503:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941609.348524:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941609.348527:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941609.348533:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:1.0:1773941609.348538:0:2393:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941609.348546:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10835180 x1860112291498496/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941619 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941609.348560:0:2393:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d10835180 x1860112291498496/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941619 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941609.348596:0:2393:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1773941609.348609:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941609.348617:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941609.348643:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941609.348649:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941609.348656:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941609.348658:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941609.348661:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941609.348669:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941609.348683:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941609.348688:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941609.348697:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10835180 x1860112291498496/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941619 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941609.348711:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10835180 x1860112291498496/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941619 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941609.348722:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941609.348724:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941609.348727:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941609.348729:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941609.348731:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941609.348733:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941609.348736:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941609.348741:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=110 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941609.348745:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941609.348748:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941609.348751:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941609.348754:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941609.348757:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941609.348760:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941609.348765:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10835180 x1860112291498496/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941619 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941609.348775:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941609.348778:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941609.348780:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941609.348782:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941609.348788:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941609.348791:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941609.348793:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941609.348796:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941609.348798:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941609.348801:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941609.348805:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941609.348809:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941609 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941609.348820:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941609.348829:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941609.348831:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941609.348834:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941609.348837:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941609.348839:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941609.348844:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941609.348847:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941609.348850:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941609.348854:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941609.348858:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941609.348862:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941609.349308:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941609.349313:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941609.349320:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10835180 x1860112291498496/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941619 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941609.349330:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10835180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291498496:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941609.349335:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941609.349339:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1773941609.349339:0:14811:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000040:3.0:1773941609.349346:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10835180 x1860112291498496/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941619 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1773941609.349352:0:14811:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00000001:3.0:1773941609.349359:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00100000:1.0:1773941609.349361:0:14811:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000001:3.0:1773941609.349363:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:1.0:1773941609.349365:0:14811:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000010:3.0:1773941609.349367:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03522400. 10000000:01000000:1.0:1773941609.349368:0:14811:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000001:3.0:1773941609.349373:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000001:00000001:1.0:1773941609.349374:0:14811:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:3.0:1773941609.349378:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000001:00000001:1.0:1773941609.349380:0:14811:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000020:00000040:3.0:1773941609.349381:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000001:00000001:1.0:1773941609.349385:0:14811:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941609.349386:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941609.349392:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d02ce4800. 00000001:00000001:1.0:1773941609.349393:0:14811:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941609.349396:0:14811:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:3.0:1773941609.349398:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000001:00000001:1.0:1773941609.349400:0:14811:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941609.349402:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 10000000:00000001:1.0:1773941609.349404:0:14811:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941609.349406:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10835180. 00000100:00000001:1.0:1773941609.349415:0:14811:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941609.349422:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00080000:1.0:1773941609.349422:0:14811:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:3.0:1773941609.349425:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941609.349427:0:14811:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:3.0:1773941609.349430:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941609.349431:0:14811:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941609.349435:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941609.349435:0:14811:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941609.349438:0:14811:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:3.0:1773941609.349439:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941609.349442:0:14811:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:3.0:1773941609.349444:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 10000000:01000000:1.0:1773941609.349445:0:14811:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000100:00000001:3.0:1773941609.349448:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 10000000:00000040:1.0:1773941609.349449:0:14811:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:1.0:1773941609.349467:0:14811:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941609.349469:0:14811:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941609.349470:0:14811:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941609.349473:0:14811:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:1.0:1773941609.349475:0:14811:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941609.349478:0:14811:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941609.349479:0:14811:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941609.349481:0:14811:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941609.349483:0:14811:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941609.467206:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941609.467212:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941609.467216:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:1.0:1773941610.427124:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:1.0:1773941610.427132:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:1.0:1773941610.427138:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941610.427141:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:1.0:1773941610.427143:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:1.0:1773941610.427145:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:1.0:1773941610.427147:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:1.0:1773941610.427149:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:1.0:1773941610.427152:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:1.0:1773941610.427155:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:1.0:1773941610.427157:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:1.0:1773941610.427169:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c2d80. 00000020:00000040:1.0:1773941610.427174:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941610.427180:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941610.427183:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941610.427188:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941610.427196:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d181fce00. 00000100:00000001:1.0:1773941610.427204:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:1.0:1773941610.427224:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:1.0:1773941610.427227:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:1.0:1773941610.427235:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d04dd4800. 00010000:00000001:1.0:1773941610.427241:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:1.0:1773941610.427244:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d01488e00. 00000020:00000001:1.0:1773941610.427249:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:1.0:1773941610.427252:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d01488e00 with handle 0x8e86767622e2ad3 to hash 00000020:00000001:1.0:1773941610.427254:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:1.0:1773941610.427256:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330112183808 : -113743597367808 : ffff988d01488e00) 00010000:00000001:1.0:1773941610.427259:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330112183808 : -113743597367808 : ffff988d01488e00) 00010000:00000001:1.0:1773941610.427263:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:1.0:1773941610.427267:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941610.427272:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2ad3 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941610.427284:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2ad3 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:1.0:1773941610.427296:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2ad3 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:1.0:1773941610.427303:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:1.0:1773941610.427306:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:1.0:1773941610.427313:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d1a1ab9c0. 00000100:00000001:1.0:1773941610.427315:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330528610752 : -113743180940864 : ffff988d1a1ab9c0) 00000100:00000001:1.0:1773941610.427318:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941610.427320:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631466368 : -113743078085248 : ffff988d203c2d80) 00000100:00000001:1.0:1773941610.427324:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941610.427329:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941610.427332:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:1.0:1773941610.427334:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941610.427344:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c2d80 x1860112291498624/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941610.427355:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941610.427364:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941610.427369:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941610.427371:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941610.427374:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941610.427379:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c2d80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291498624:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:1.0:1773941610.427383:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941610.427389:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941610.427392:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941610.427394:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941610.427399:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d11189400. 02000000:00000001:1.0:1773941610.427402:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941610.427410:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941610.427414:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c7e8. 00000400:00000010:1.0:1773941610.427420:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de088. 00000100:00000200:1.0:1773941610.427427:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291498624, portal 25 00000100:00000001:1.0:1773941610.427429:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941610.427430:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631466368 : -113743078085248 : ffff988d203c2d80) 00000100:00000040:1.0:1773941610.427436:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c2d80 x1860112291498624/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941626 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941610.427445:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941610.427448:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941610.427451:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113deee0. 00000100:00000200:1.0:1773941610.427454:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291498624, offset 0 00000400:00000200:1.0:1773941610.427460:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941610.427468:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941610.427476:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941610.427481:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941610.427486:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941610.427491:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941610.427494:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941610.427497:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[172, 173] 00000400:00000200:1.0:1773941610.427502:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:1.0:1773941610.427506:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [284:284:256:4294967295] 192.168.204.114@tcp LPNI seq info [174:174:8:4294967295] 00000400:00000200:1.0:1773941610.427517:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941610.427528:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:1.0:1773941610.427535:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:1.0:1773941610.427540:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfd800. 00000800:00000200:1.0:1773941610.427547:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:1.0:1773941610.427556:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:1.0:1773941610.427561:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfd800 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:1.0:1773941610.427581:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941610.427586:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941610.427589:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4295002023 00000100:00000001:1.0:1773941610.427596:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4295000938 00000100:00000001:1.0:1773941610.427601:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4295002023 00000100:00000001:1.0:1773941610.427604:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4295000938 00000100:00000001:1.0:1773941610.427608:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941610.427611:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941610.427613:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941610.427616:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:1.0:1773941610.427618:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 11 seconds 00000100:00000001:1.0:1773941610.427621:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941610.427624:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941610.427740:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941610.427745:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfd800. 00000400:00000200:0.0:1773941610.427750:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941610.427755:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941610.427759:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113deee0 00000400:00000010:0.0:1773941610.427761:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113deee0. 00000100:00000001:0.0:1773941610.427764:0:2394:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941610.427769:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c2d80 x1860112291498624/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941626 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941610.427777:0:2394:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941610.427780:0:2394:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d203c2d80 x1860112291498624/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941626 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941610.427785:0:2394:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941610.427787:0:2394:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:1.0:1773941610.429117:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941610.429132:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941610.429135:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941610.429138:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941610.429143:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941610.429149:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941610.429156:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941610.429160:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941610.429165:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941610.429169:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324608a80 00000400:00000200:1.0:1773941610.429173:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941610.429184:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0x1229 [1] + 224 00000800:00000001:1.0:1773941610.429191:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941610.429199:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941610.429202:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941610.429206:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:1.0:1773941610.429211:0:2393:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941610.429216:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c2d80 x1860112291498624/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941626 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941610.429228:0:2393:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d203c2d80 x1860112291498624/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941626 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941610.429248:0:2393:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1773941610.429256:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941610.429259:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941610.429280:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941610.429286:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941610.429288:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941610.429290:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c7e8. 00000400:00000200:0.0:1773941610.429294:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de088 00000400:00000010:0.0:1773941610.429295:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de088. 00000100:00000001:0.0:1773941610.429304:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941610.429308:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c2d80 x1860112291498624/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941626 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941610.429316:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c2d80 x1860112291498624/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941626 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941610.429321:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941610.429323:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941610.429325:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941610.429327:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941610.429329:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941610.429330:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941610.429332:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941610.429338:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=111 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941610.429340:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941610.429342:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:0.0:1773941610.429345:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:0.0:1773941610.429347:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941610.429349:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:0.0:1773941610.429352:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:0.0:1773941610.429355:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:0.0:1773941610.429358:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:0.0:1773941610.429361:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:0.0:1773941610.429363:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:0.0:1773941610.429364:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:0.0:1773941610.429365:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:0.0:1773941610.429368:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d203c2d80 x1860112291498624/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941626 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941610.429373:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:0.0:1773941610.429374:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:0.0:1773941610.429375:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941610.429377:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:0.0:1773941610.429379:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:0.0:1773941610.429383:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941610.429385:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941610.429386:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941610.429387:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:0.0:1773941610.429388:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941610.429389:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941610.429391:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1773941610.429392:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1773941610.429394:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1773941610.429397:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 332 00000400:00000200:0.0:1773941610.429400:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941610.429403:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:0.0:1773941610.429406:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941610.429409:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941610.429410:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941610.429412:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:0.0:1773941610.429414:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941610.429416:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:0.0:1773941610.429417:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941610.429419:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:0.0:1773941610.429421:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:0.0:1773941610.429423:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:0.0:1773941610.429425:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941610.429426:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941610.429427:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941610.429428:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:0.0:1773941610.429430:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:0.0:1773941610.429432:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:0.0:1773941610.429434:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941610.429436:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1773941610.429438:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941610.429439:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941610.429444:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:0.0:1773941610.429446:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1773941610.429449:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10836a00. 00000020:00000040:0.0:1773941610.429451:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:0.0:1773941610.429453:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1773941610.429454:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941610.429456:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941610.429457:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941610.429459:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941610.429462:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d02ce6800. 00000100:00000001:0.0:1773941610.429465:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941610.429469:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d10836a00 x1860112291498752/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1773941610.429475:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941610.429477:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941610.429480:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f9e7d804] to pc [ptlrpcd_rcv+-1] req@ffff988d10836a00 x1860112291498752/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941610.429497:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941610.429499:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:0.0:1773941610.429500:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941610.429502:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941610.429504:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941610.429505:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1773941610.429507:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d1a1ab9c0 going to sleep for 1 seconds 00000100:00000001:0.0:1773941610.429508:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941610.429509:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1773941610.429512:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c2d80 x1860112291498624/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941626 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941610.429518:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:0.0:1773941610.429521:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941610.429523:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941610.429524:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1773941610.429527:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c2d80 x1860112291498624/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941626 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941610.429531:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941610.429537:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941610.429540:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00100000:0.0:1773941610.429541:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c2d80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291498624:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941610.429546:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941610.429549:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:3.0:1773941610.429551:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10836a00 x1860112291498752/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941610.429551:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1773941610.429554:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941610.429559:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c2d80 x1860112291498624/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941626 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941610.429577:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941610.429577:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941610.429580:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1773941610.429581:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d1a1ab9c0. 02000000:00000001:3.0:1773941610.429588:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1773941610.429588:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1773941610.429592:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:3.0:1773941610.429594:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1773941610.429595:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1773941610.429599:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 02000000:00000001:3.0:1773941610.429600:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941610.429601:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941610.429605:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488e00 refcount=4 00000020:00000001:0.0:1773941610.429607:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112183808 : -113743597367808 : ffff988d01488e00) 00000100:00100000:3.0:1773941610.429608:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10836a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291498752:192.168.204.114@tcp:250:ll_cfg_requeue.0 00010000:00000001:0.0:1773941610.429610:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112183808 : -113743597367808 : ffff988d01488e00) 00010000:00010000:0.0:1773941610.429615:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2ad3 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941610.429617:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941610.429621:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00000001:0.0:1773941610.429624:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 02000000:00000001:3.0:1773941610.429625:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941610.429629:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2ad3 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941610.429630:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941610.429638:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03522400. 00010000:00000001:0.0:1773941610.429638:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1773941610.429641:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2ad3 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941610.429643:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941610.429650:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:3.0:1773941610.429652:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:0.0:1773941610.429657:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000010:3.0:1773941610.429658:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00010000:00010000:0.0:1773941610.429660:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2ad3 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:3.0:1773941610.429666:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00010000:00000001:0.0:1773941610.429667:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941610.429669:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1773941610.429673:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291498752, portal 25 00010000:00000001:0.0:1773941610.429674:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1773941610.429677:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2ad3 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941610.429679:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941610.429682:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367699456 : -113743341852160 : ffff988d10836a00) 00010000:00010000:0.0:1773941610.429684:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d01488e00 already unused, calling callback (ffffffffc12b4c20) 10000000:00000001:0.0:1773941610.429688:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1773941610.429694:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2ad3 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:3.0:1773941610.429695:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10836a00 x1860112291498752/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941620 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1773941610.429706:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1773941610.429709:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1773941610.429711:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941610.429714:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d01488e00 refcount=5 00000100:00000001:3.0:1773941610.429716:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:0.0:1773941610.429717:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330112183808 : -113743597367808 : ffff988d01488e00) 00010000:00000001:0.0:1773941610.429721:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330112183808 : -113743597367808 : ffff988d01488e00) 00000100:00000040:3.0:1773941610.429723:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00000001:0.0:1773941610.429725:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1773941610.429727:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2ad3 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1773941610.429734:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000010:3.0:1773941610.429736:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac088. 10000000:00010000:0.0:1773941610.429737:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2ad3 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:3.0:1773941610.429743:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291498752, offset 0 10000000:01000000:0.0:1773941610.429744:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:0.0:1773941610.429748:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1773941610.429751:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941610.429753:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000400:00000200:3.0:1773941610.429755:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00000001:0.0:1773941610.429756:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:3.0:1773941610.429764:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:0.0:1773941610.429766:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1773941610.429769:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1773941610.429770:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:3.0:1773941610.429772:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:0.0:1773941610.429773:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1773941610.429779:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d01488e00 with handle 0x8e86767622e2ad3 from hash 00000400:00000200:3.0:1773941610.429780:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:0.0:1773941610.429783:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000400:00000200:3.0:1773941610.429787:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00000001:0.0:1773941610.429787:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941610.429789:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941610.429792:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000400:00000200:3.0:1773941610.429793:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:0.0:1773941610.429794:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1773941610.429797:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000400:00000200:3.0:1773941610.429800:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:0.0:1773941610.429801:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941610.429804:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941610.429807:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941610.429810:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[174, 173] 10000000:00000001:0.0:1773941610.429810:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941610.429816:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2ad3 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941610.429819:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00010000:00000001:0.0:1773941610.429824:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:3.0:1773941610.429827:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [285:285:256:4294967295] 192.168.204.124@tcp LPNI seq info [175:175:8:4294967295] 00010000:00000001:0.0:1773941610.429827:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941610.429830:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1773941610.429832:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1773941610.429835:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941610.429837:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:3.0:1773941610.429839:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00010000:00000001:0.0:1773941610.429839:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1773941610.429843:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d01488e00/0x8e86767622e2ad3 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1773941610.429852:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d04dd4800 count: 0 00000400:00000200:3.0:1773941610.429853:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:3.0:1773941610.429861:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.124@tcp 00010000:00000001:0.0:1773941610.429861:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941610.429865:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941610.429867:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000800:00000010:3.0:1773941610.429868:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74800. 00000100:00000040:0.0:1773941610.429871:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c2d80 x1860112291498624/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941626 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000800:00000200:3.0:1773941610.429879:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000100:00000001:0.0:1773941610.429882:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1773941610.429885:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000800:00000200:3.0:1773941610.429889:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 02000000:00000010:0.0:1773941610.429892:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d11189400. 00000800:00000200:3.0:1773941610.429894:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74800 type 1, nob 616 niov 1 nkiov 1 02000000:00000001:0.0:1773941610.429896:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1773941610.429899:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941610.429901:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:0.0:1773941610.429903:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1773941610.429905:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d181fce00. 02000000:00000001:0.0:1773941610.429907:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1773941610.429908:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1773941610.429910:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1773941610.429910:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c2d80. 00000100:00000001:0.0:1773941610.429914:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941610.429917:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:3.0:1773941610.429918:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 10000000:00000001:0.0:1773941610.429920:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941610.429923:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4295002025 10000000:01000000:0.0:1773941610.429924:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:0.0:1773941610.429928:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000100:00000001:3.0:1773941610.429931:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4295002023 10000000:00000010:0.0:1773941610.429932:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d10872940. 00000020:00000010:0.0:1773941610.429938:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d065c5a00. 00000100:00000001:3.0:1773941610.429940:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4295002025 00000020:00000010:0.0:1773941610.429943:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d065c5c00. 00000100:00000001:3.0:1773941610.429945:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4295002023 00000040:00000010:0.0:1773941610.429947:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d065c4200. 00000100:00000001:3.0:1773941610.429951:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1773941610.429952:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d054a7000. 00000100:00000001:3.0:1773941610.429956:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941610.429960:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941610.429984:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941610.429991:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000020:00000010:0.0:1773941610.429998:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d065c4a00. 00000100:00000001:3.0:1773941610.429999:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1773941610.430005:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 80000000:00000010:0.0:1773941610.430006:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d06580420. 00000100:00000001:3.0:1773941610.430007:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941610.430011:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00040000:00000010:0.0:1773941610.430013:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d065c5400. 00000100:00000001:3.0:1773941610.430014:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 20000000:00000010:0.0:1773941610.430019:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d065c5e00. 10000000:00000040:0.0:1773941610.430024:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:0.0:1773941610.430026:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000800:00000200:1.0:1773941610.430028:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000020:00000010:0.0:1773941610.430030:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d065c5c00. 00000800:00000010:1.0:1773941610.430035:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74800. 00000040:00000010:0.0:1773941610.430037:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d065c4200. 00000400:00000200:1.0:1773941610.430044:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000020:00000010:0.0:1773941610.430044:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d054a7000. 00000020:00000010:0.0:1773941610.430051:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d065c4a00. 80000000:00000010:0.0:1773941610.430055:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d06580420. 00000400:00000200:1.0:1773941610.430056:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00040000:00000010:0.0:1773941610.430059:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d065c5400. 00000400:00000200:1.0:1773941610.430063:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac088 20000000:00000010:0.0:1773941610.430063:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d065c5e00. 00000400:00000010:1.0:1773941610.430068:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac088. 00000020:00000010:0.0:1773941610.430069:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d065c5a00. 10000000:00000010:0.0:1773941610.430073:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d10872940. 00000100:00000001:1.0:1773941610.430077:0:2394:0:(events.c:34:request_out_callback()) Process entered 10000000:01000000:0.0:1773941610.430077:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 00000100:00000200:1.0:1773941610.430084:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10836a00 x1860112291498752/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941620 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000001:0.0:1773941610.430086:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:0.0:1773941610.430091:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:0.0:1773941610.430094:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1773941610.430097:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1773941610.430114:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941610.430118:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000100:00000001:1.0:1773941610.430119:0:2394:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941610.430126:0:2394:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10836a00 x1860112291498752/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941620 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941610.430135:0:2394:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941610.430137:0:2394:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:1.0:1773941610.431511:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941610.431518:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941610.431521:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941610.431523:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941610.431528:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941610.431538:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:1.0:1773941610.431543:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941610.431546:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941610.431550:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941610.431553:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324608b00 00000400:00000200:1.0:1773941610.431557:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941610.431560:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0x1239 [1] + 0 00000800:00000001:1.0:1773941610.431565:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941610.431571:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941610.431573:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941610.431577:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:1.0:1773941610.431581:0:2393:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941610.431586:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10836a00 x1860112291498752/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941620 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941610.431596:0:2393:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d10836a00 x1860112291498752/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941620 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941610.431611:0:2393:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1773941610.431616:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941610.431619:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941610.431642:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941610.431645:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941610.431649:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941610.431651:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941610.431653:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941610.431658:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941610.431660:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941610.431663:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941610.431669:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10836a00 x1860112291498752/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941620 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941610.431679:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10836a00 x1860112291498752/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941620 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941610.431687:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941610.431689:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941610.431691:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941610.431693:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941610.431694:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941610.431696:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941610.431698:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941610.431702:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=111 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941610.431705:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941610.431708:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941610.431709:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941610.431711:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941610.431713:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941610.431716:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941610.431720:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10836a00 x1860112291498752/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941620 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941610.431727:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941610.431729:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941610.431737:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941610.431739:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941610.431743:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941610.431745:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941610.431746:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941610.431748:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941610.431750:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941610.431752:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941610.431755:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941610.431759:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941610 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941610.431761:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941610.431767:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941610.431769:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941610.431771:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941610.431773:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941610.431774:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941610.431777:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941610.431780:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941610.431782:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941610.431785:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941610.431788:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941610.431791:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941610.431993:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941610.431997:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941610.432004:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10836a00 x1860112291498752/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941620 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941610.432012:0:14812:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00100000:3.0:1773941610.432014:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10836a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291498752:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941610.432022:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1773941610.432024:0:14812:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00000001:3.0:1773941610.432026:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00100000:1.0:1773941610.432029:0:14812:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000040:3.0:1773941610.432034:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10836a00 x1860112291498752/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941620 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941610.432035:0:14812:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941610.432038:0:14812:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:1.0:1773941610.432043:0:14812:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941610.432046:0:14812:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941610.432050:0:14812:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941610.432054:0:14812:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941610.432057:0:14812:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941610.432060:0:14812:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941610.432065:0:14812:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941610.432069:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:1.0:1773941610.432069:0:14812:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000001:3.0:1773941610.432074:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00080000:1.0:1773941610.432077:0:14812:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 02000000:00000010:3.0:1773941610.432079:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03522400. 02000000:00000001:3.0:1773941610.432083:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941610.432086:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:1.0:1773941610.432089:0:14812:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000020:00000040:3.0:1773941610.432090:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941610.432094:0:14812:0:(recover.c:147:ptlrpc_resend()) Process entered 00000020:00000001:3.0:1773941610.432095:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941610.432112:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d02ce6800. 00000100:00000001:1.0:1773941610.432113:0:14812:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941610.432116:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00080000:1.0:1773941610.432119:0:14812:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 02000000:00000001:3.0:1773941610.432120:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941610.432124:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10836a00. 00000100:00000001:1.0:1773941610.432126:0:14812:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:3.0:1773941610.432129:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 10000000:01000000:1.0:1773941610.432130:0:14812:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 00000100:00000001:3.0:1773941610.432133:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000040:1.0:1773941610.432135:0:14812:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00000001:3.0:1773941610.432139:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941610.432146:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941610.432148:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941610.432150:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941610.432152:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941610.432153:0:14812:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941610.432156:0:14812:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941610.432158:0:14812:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941610.432164:0:14812:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:1.0:1773941610.432169:0:14812:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941610.432172:0:14812:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941610.432175:0:14812:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941610.432177:0:14812:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941610.432179:0:14812:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:0.1:1773941610.435011:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d04dd4800. 00010000:00000010:0.1:1773941610.435017:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d01488e00. 00000020:00000001:1.0:1773941610.491148:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1773941610.491155:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941610.491160:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1773941611.515270:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941611.515279:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941611.515284:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941611.707117:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4295002025, at 4295002048 00000100:00000040:0.0:1773941611.707139:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4295002025, at 4295002048 00000100:00000010:1.0:1773941611.707140:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580a20. 00000100:00000010:0.0:1773941611.707157:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580b40. 10000000:00000001:0.0:1773941612.411078:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1773941612.411085:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941612.411088:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941612.411090:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941612.411091:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941612.411092:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1773941612.411094:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:0.0:1773941612.411095:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:0.0:1773941612.411097:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:0.0:1773941612.411099:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1773941612.411101:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:0.0:1773941612.411110:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10837b80. 00000020:00000040:0.0:1773941612.411115:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941612.411119:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941612.411122:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941612.411126:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941612.411133:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065c5a00. 00000100:00000001:0.0:1773941612.411138:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941612.411142:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1773941612.411145:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1773941612.411151:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d21ec1400. 00010000:00000001:0.0:1773941612.411156:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1773941612.411159:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d00031600. 00000020:00000001:0.0:1773941612.411162:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:0.0:1773941612.411163:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d00031600 with handle 0x8e86767622e2ada to hash 00000020:00000001:0.0:1773941612.411165:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:0.0:1773941612.411166:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330090853888 : -113743618697728 : ffff988d00031600) 00010000:00000001:0.0:1773941612.411168:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330090853888 : -113743618697728 : ffff988d00031600) 00010000:00000001:0.0:1773941612.411185:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941612.411188:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941612.411191:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ada lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941612.411199:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ada lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941612.411209:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ada lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941612.411213:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1773941612.411215:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941612.411220:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb6840. 00000100:00000001:0.0:1773941612.411222:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330139633728 : -113743569917888 : ffff988d02eb6840) 00000100:00000001:0.0:1773941612.411224:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941612.411225:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367703936 : -113743341847680 : ffff988d10837b80) 00000100:00000001:0.0:1773941612.411229:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941612.411232:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941612.411234:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941612.411235:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941612.411243:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10837b80 x1860112291498880/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941612.411250:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941612.411252:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941612.411255:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941612.411257:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941612.411258:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941612.411261:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10837b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291498880:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941612.411264:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941612.411269:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941612.411270:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941612.411272:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941612.411278:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d02ce7000. 02000000:00000001:0.0:1773941612.411280:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941612.411286:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941612.411289:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0014baa8. 00000400:00000010:0.0:1773941612.411294:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d108336e8. 00000100:00000200:0.0:1773941612.411299:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291498880, portal 25 00000100:00000001:0.0:1773941612.411301:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941612.411302:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367703936 : -113743341847680 : ffff988d10837b80) 00000100:00000040:0.0:1773941612.411306:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10837b80 x1860112291498880/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941628 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941612.411311:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941612.411313:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941612.411315:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833110. 00000100:00000200:0.0:1773941612.411317:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291498880, offset 0 00000400:00000200:0.0:1773941612.411322:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941612.411329:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941612.411335:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941612.411339:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941612.411342:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941612.411345:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941612.411347:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941612.411350:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[174, 175] 00000400:00000200:0.0:1773941612.411353:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:0.0:1773941612.411356:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [286:286:256:4294967295] 192.168.204.114@tcp LPNI seq info [176:176:8:4294967295] 00000400:00000200:0.0:1773941612.411359:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941612.411365:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:0.0:1773941612.411370:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:0.0:1773941612.411373:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d113e9a00. 00000800:00000200:0.0:1773941612.411378:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:0.0:1773941612.411385:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:0.0:1773941612.411388:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d113e9a00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1773941612.411408:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941612.411411:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941612.411413:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4295004007 00000100:00000010:0.0:1773941612.411416:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580b40. 00000100:00000001:0.0:1773941612.411422:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1773941612.411428:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4295004007 00000100:00000010:0.0:1773941612.411430:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580420. 00000100:00000001:0.0:1773941612.411432:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1773941612.411437:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941612.411439:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941612.411441:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941612.411443:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1773941612.411445:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6840 going to sleep for 11 seconds 00000100:00000001:0.0:1773941612.411448:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941612.411450:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941612.411536:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941612.411541:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d113e9a00. 00000400:00000200:1.0:1773941612.411546:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941612.411551:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941612.411555:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833110 00000400:00000010:1.0:1773941612.411557:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833110. 00000100:00000001:1.0:1773941612.411560:0:2394:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941612.411564:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10837b80 x1860112291498880/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941628 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941612.411572:0:2394:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941612.411575:0:2394:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d10837b80 x1860112291498880/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941628 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941612.411580:0:2394:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941612.411588:0:2394:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:1.0:1773941612.412728:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941612.412740:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941612.412743:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941612.412744:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941612.412748:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941612.412753:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941612.412758:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941612.412761:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941612.412765:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941612.412769:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324608b80 00000400:00000200:1.0:1773941612.412772:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941612.412775:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0x1241 [1] + 224 00000800:00000001:1.0:1773941612.412781:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941612.412787:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941612.412789:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941612.412791:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:1.0:1773941612.412794:0:2393:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941612.412798:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10837b80 x1860112291498880/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941628 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941612.412806:0:2393:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d10837b80 x1860112291498880/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941628 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941612.412820:0:2393:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1773941612.412826:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941612.412829:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941612.412853:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941612.412860:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941612.412861:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941612.412863:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0014baa8. 00000400:00000200:0.0:1773941612.412867:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d108336e8 00000400:00000010:0.0:1773941612.412868:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d108336e8. 00000100:00000001:0.0:1773941612.412870:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941612.412875:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10837b80 x1860112291498880/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941628 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941612.412883:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10837b80 x1860112291498880/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941628 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941612.412889:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941612.412890:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941612.412892:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941612.412894:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941612.412896:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941612.412898:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941612.412900:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941612.412905:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=113 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941612.412913:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941612.412915:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:0.0:1773941612.412919:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:0.0:1773941612.412921:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941612.412924:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:0.0:1773941612.412926:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:0.0:1773941612.412929:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:0.0:1773941612.412933:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:0.0:1773941612.412935:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:0.0:1773941612.412937:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:0.0:1773941612.412938:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:0.0:1773941612.412940:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:0.0:1773941612.412943:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d10837b80 x1860112291498880/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941628 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941612.412951:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:0.0:1773941612.412952:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:0.0:1773941612.412954:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941612.412957:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:0.0:1773941612.412981:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:0.0:1773941612.412983:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941612.412986:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941612.412987:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941612.412987:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:0.0:1773941612.412989:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941612.412990:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941612.412992:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1773941612.412993:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1773941612.412995:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1773941612.412997:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 333 00000400:00000200:0.0:1773941612.413001:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941612.413005:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:0.0:1773941612.413007:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941612.413010:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941612.413017:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941612.413018:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:0.0:1773941612.413021:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941612.413022:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:0.0:1773941612.413024:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941612.413026:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:0.0:1773941612.413027:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:0.0:1773941612.413029:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:0.0:1773941612.413031:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941612.413032:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941612.413033:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941612.413034:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:0.0:1773941612.413036:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:0.0:1773941612.413039:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:0.0:1773941612.413040:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941612.413042:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1773941612.413044:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941612.413046:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941612.413047:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:0.0:1773941612.413049:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1773941612.413051:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10835880. 00000020:00000040:0.0:1773941612.413053:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:0.0:1773941612.413055:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1773941612.413056:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941612.413057:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941612.413059:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941612.413061:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941612.413079:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d02ce7400. 00000100:00000001:0.0:1773941612.413083:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941612.413088:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d10835880 x1860112291499008/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1773941612.413094:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941612.413100:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941612.413104:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c1418601] to pc [ptlrpcd_rcv+-1] req@ffff988d10835880 x1860112291499008/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941612.413119:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941612.413121:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:0.0:1773941612.413122:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941612.413125:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941612.413126:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941612.413127:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1773941612.413129:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6840 going to sleep for 1 seconds 00000100:00000001:0.0:1773941612.413130:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941612.413132:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1773941612.413135:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10837b80 x1860112291498880/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941628 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941612.413141:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:0.0:1773941612.413143:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941612.413145:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941612.413146:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941612.413149:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000040:0.0:1773941612.413150:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10837b80 x1860112291498880/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941628 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941612.413154:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941612.413157:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00100000:0.0:1773941612.413161:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10837b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291498880:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000040:3.0:1773941612.413167:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10835880 x1860112291499008/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941612.413167:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941612.413171:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941612.413176:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1773941612.413178:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1773941612.413183:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000040:0.0:1773941612.413184:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10837b80 x1860112291498880/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941628 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941612.413187:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941612.413192:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941612.413195:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941612.413196:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941612.413201:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1773941612.413202:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb6840. 00000100:00000001:0.0:1773941612.413206:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00100000:3.0:1773941612.413207:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10835880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291499008:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941612.413211:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1773941612.413215:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:3.0:1773941612.413216:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941612.413218:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941612.413220:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 02000000:00000001:3.0:1773941612.413221:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00010000:00000001:0.0:1773941612.413224:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1773941612.413226:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 02000000:00000010:3.0:1773941612.413227:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03522400. 00000020:00000040:0.0:1773941612.413228:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00031600 refcount=4 00000020:00000001:0.0:1773941612.413230:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090853888 : -113743618697728 : ffff988d00031600) 00010000:00000001:0.0:1773941612.413232:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090853888 : -113743618697728 : ffff988d00031600) 00010000:00010000:0.0:1773941612.413234:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ada lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941612.413237:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941612.413246:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:0.0:1773941612.413246:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000400:00000010:3.0:1773941612.413251:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00010000:00010000:0.0:1773941612.413253:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ada lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:3.0:1773941612.413256:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00010000:00000001:0.0:1773941612.413262:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000200:3.0:1773941612.413263:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291499008, portal 25 00000100:00000001:3.0:1773941612.413267:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00010000:0.0:1773941612.413267:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ada lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941612.413268:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367694976 : -113743341856640 : ffff988d10835880) 00010000:00000001:0.0:1773941612.413277:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000040:3.0:1773941612.413278:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10835880 x1860112291499008/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941622 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1773941612.413280:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1773941612.413283:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ada lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1773941612.413298:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:3.0:1773941612.413300:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:0.0:1773941612.413301:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941612.413305:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000100:00000040:3.0:1773941612.413307:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00010000:0.0:1773941612.413308:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ada lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:3.0:1773941612.413314:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac5d8. 00010000:00010000:0.0:1773941612.413316:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d00031600 already unused, calling callback (ffffffffc12b4c20) 10000000:00000001:0.0:1773941612.413320:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000200:3.0:1773941612.413321:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291499008, offset 0 10000000:00010000:0.0:1773941612.413322:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ada lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941612.413332:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00000001:0.0:1773941612.413338:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1773941612.413341:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000200:3.0:1773941612.413342:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000020:00000001:0.0:1773941612.413343:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941612.413346:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00031600 refcount=5 00000020:00000001:0.0:1773941612.413349:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090853888 : -113743618697728 : ffff988d00031600) 00010000:00000001:0.0:1773941612.413350:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090853888 : -113743618697728 : ffff988d00031600) 00000400:00000200:3.0:1773941612.413352:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:0.0:1773941612.413354:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000400:00000200:3.0:1773941612.413357:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00010000:0.0:1773941612.413358:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ada lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941612.413362:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941612.413368:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 10000000:00000001:0.0:1773941612.413370:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:3.0:1773941612.413373:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 10000000:00010000:0.0:1773941612.413374:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ada lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941612.413379:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[176, 175] 10000000:01000000:0.0:1773941612.413388:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000400:00000200:3.0:1773941612.413389:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 10000000:00000040:0.0:1773941612.413394:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000400:00000200:3.0:1773941612.413396:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [287:287:256:4294967295] 192.168.204.124@tcp LPNI seq info [177:177:8:4294967295] 10000000:00000001:0.0:1773941612.413397:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941612.413403:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000400:00000200:3.0:1773941612.413406:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00010000:00000001:0.0:1773941612.413407:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1773941612.413410:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1773941612.413414:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1773941612.413417:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:3.0:1773941612.413418:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00010000:00000001:0.0:1773941612.413419:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1773941612.413425:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d00031600 with handle 0x8e86767622e2ada from hash 00000800:00000200:3.0:1773941612.413428:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.124@tcp 00010000:00000001:0.0:1773941612.413429:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1773941612.413432:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941612.413434:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000010:3.0:1773941612.413435:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74c00. 00010000:00000001:0.0:1773941612.413436:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1773941612.413439:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1773941612.413441:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000800:00000200:3.0:1773941612.413445:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00010000:00000001:0.0:1773941612.413445:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941612.413447:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941612.413450:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941612.413454:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 10000000:00000001:0.0:1773941612.413458:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941612.413459:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74c00 type 1, nob 616 niov 1 nkiov 1 00010000:00010000:0.0:1773941612.413463:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ada lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1773941612.413468:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941612.413469:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941612.413470:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1773941612.413471:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000100:00000001:3.0:1773941612.413473:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941612.413473:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941612.413475:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941612.413476:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1773941612.413477:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ada lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941612.413486:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000040:0.0:1773941612.413487:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d21ec1400 count: 0 00000100:00000001:3.0:1773941612.413489:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4295004009 00000100:00000001:3.0:1773941612.413496:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4295004007 00000100:00000001:3.0:1773941612.413501:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4295004009 00010000:00000001:0.0:1773941612.413501:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941612.413505:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941612.413506:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4295004007 00000100:00000001:0.0:1773941612.413510:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1773941612.413512:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941612.413517:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941612.413517:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10837b80 x1860112291498880/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941628 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941612.413522:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941612.413527:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941612.413531:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1773941612.413533:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1773941612.413535:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 02000000:00000010:0.0:1773941612.413536:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d02ce7000. 00000100:00000001:3.0:1773941612.413538:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 02000000:00000001:0.0:1773941612.413540:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1773941612.413543:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1773941612.413543:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941612.413545:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941612.413547:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1773941612.413549:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:3.0:1773941612.413553:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941612.413553:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065c5a00. 00000100:00000001:3.0:1773941612.413559:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941612.413560:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1773941612.413563:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1773941612.413566:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10837b80. 00000800:00000200:1.0:1773941612.413568:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1773941612.413569:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941612.413573:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000800:00000010:1.0:1773941612.413574:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74c00. 10000000:00000001:0.0:1773941612.413575:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:0.0:1773941612.413580:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000400:00000200:1.0:1773941612.413581:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 10000000:00000001:0.0:1773941612.413583:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000400:00000200:1.0:1773941612.413588:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 10000000:00000010:0.0:1773941612.413592:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d10872940. 00000400:00000200:1.0:1773941612.413593:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac5d8 00000400:00000010:1.0:1773941612.413598:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac5d8. 00000020:00000010:0.0:1773941612.413599:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d065c5a00. 00000100:00000001:1.0:1773941612.413601:0:2394:0:(events.c:34:request_out_callback()) Process entered 00000020:00000010:0.0:1773941612.413604:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d065c5e00. 00000100:00000200:1.0:1773941612.413608:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10835880 x1860112291499008/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941622 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000040:00000010:0.0:1773941612.413610:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d065c5400. 00000020:00000010:0.0:1773941612.413613:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d054a7000. 00000020:00000010:0.0:1773941612.413618:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d065c4a00. 80000000:00000010:0.0:1773941612.413623:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d065803c0. 00000100:00000001:1.0:1773941612.413628:0:2394:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00040000:00000010:0.0:1773941612.413629:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d065c4200. 00000100:00000040:1.0:1773941612.413633:0:2394:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10835880 x1860112291499008/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941622 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 20000000:00000010:0.0:1773941612.413635:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d065c5c00. 10000000:00000040:0.0:1773941612.413642:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 00000100:00000001:1.0:1773941612.413644:0:2394:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941612.413645:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:1.0:1773941612.413648:0:2394:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:0.0:1773941612.413650:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d065c5e00. 00000040:00000010:0.0:1773941612.413656:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d065c5400. 00000020:00000010:0.0:1773941612.413659:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d054a7000. 00000020:00000010:0.0:1773941612.413662:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d065c4a00. 80000000:00000010:0.0:1773941612.413666:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d065803c0. 00040000:00000010:0.0:1773941612.413668:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d065c4200. 20000000:00000010:0.0:1773941612.413671:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d065c5c00. 00000020:00000010:0.0:1773941612.413673:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d065c5a00. 10000000:00000010:0.0:1773941612.413675:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d10872940. 10000000:01000000:0.0:1773941612.413676:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:0.0:1773941612.413678:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:0.0:1773941612.413679:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:0.0:1773941612.413681:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1773941612.413682:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1773941612.413683:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941612.413684:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:1.0:1773941612.414926:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941612.414935:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941612.414938:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941612.414940:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941612.414945:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941612.414949:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:1.0:1773941612.414955:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941612.414996:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941612.415001:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941612.415003:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324608c00 00000400:00000200:1.0:1773941612.415006:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941612.415009:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0x1251 [1] + 0 00000800:00000001:1.0:1773941612.415013:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941612.415018:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941612.415020:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941612.415022:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:1.0:1773941612.415025:0:2393:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941612.415029:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10835880 x1860112291499008/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941622 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941612.415038:0:2393:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d10835880 x1860112291499008/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941622 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941612.415050:0:2393:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1773941612.415054:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941612.415056:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941612.415086:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941612.415089:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941612.415093:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941612.415094:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941612.415095:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941612.415116:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941612.415118:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941612.415122:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941612.415129:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10835880 x1860112291499008/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941622 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941612.415142:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10835880 x1860112291499008/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941622 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941612.415150:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941612.415152:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941612.415155:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941612.415157:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941612.415159:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941612.415161:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941612.415179:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941612.415183:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=113 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941612.415195:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941612.415198:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941612.415199:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941612.415202:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941612.415204:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941612.415206:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941612.415211:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10835880 x1860112291499008/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941622 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941612.415219:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941612.415222:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941612.415224:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941612.415226:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941612.415230:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941612.415233:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941612.415234:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941612.415236:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941612.415238:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941612.415240:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941612.415243:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941612.415247:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941612 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941612.415250:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941612.415257:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941612.415258:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941612.415260:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941612.415261:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941612.415262:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941612.415266:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941612.415267:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941612.415269:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941612.415272:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941612.415274:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941612.415276:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941612.415484:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941612.415492:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941612.415498:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10835880 x1860112291499008/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941622 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941612.415504:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10835880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291499008:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:1.0:1773941612.415507:0:14813:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:3.0:1773941612.415509:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941612.415511:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941612.415513:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10835880 x1860112291499008/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941622 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941612.415520:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941612.415522:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941612.415525:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03522400. 00000100:00080000:1.0:1773941612.415525:0:14813:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 02000000:00000001:3.0:1773941612.415530:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00100000:1.0:1773941612.415530:0:14813:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000020:00000001:3.0:1773941612.415533:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:1.0:1773941612.415534:0:14813:0:(obd_class.h:1215:obd_import_event()) Process entered 00000020:00000040:3.0:1773941612.415536:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 10000000:01000000:1.0:1773941612.415537:0:14813:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000020:00000001:3.0:1773941612.415541:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:1.0:1773941612.415542:0:14813:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000010:3.0:1773941612.415544:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d02ce7400. 00000001:00000001:1.0:1773941612.415546:0:14813:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:3.0:1773941612.415548:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941612.415550:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000001:00000001:1.0:1773941612.415550:0:14813:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941612.415554:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10835880. 00000001:00000001:1.0:1773941612.415555:0:14813:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:3.0:1773941612.415558:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000001:00000001:1.0:1773941612.415559:0:14813:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:3.0:1773941612.415563:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:1.0:1773941612.415563:0:14813:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941612.415568:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:1.0:1773941612.415569:0:14813:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941612.415574:0:14813:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941612.415575:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:1.0:1773941612.415577:0:14813:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:3.0:1773941612.415578:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941612.415583:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941612.415584:0:14813:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:3.0:1773941612.415586:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941612.415587:0:14813:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941612.415590:0:14813:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941612.415592:0:14813:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941612.415593:0:14813:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941612.415594:0:14813:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941612.415596:0:14813:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:1.0:1773941612.415607:0:14813:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941612.415608:0:14813:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941612.415609:0:14813:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941612.415611:0:14813:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:1.0:1773941612.415613:0:14813:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941612.415615:0:14813:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941612.415616:0:14813:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941612.415618:0:14813:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941612.415619:0:14813:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:0.1:1773941612.423016:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d21ec1400. 00010000:00000010:0.1:1773941612.423023:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d00031600. 00000020:00000001:1.0:1773941612.539164:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1773941612.539170:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941612.539173:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:3.0:1773941612.795199:0:461:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (340) 00000020:00000001:2.0:1773941613.563089:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941613.563095:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941613.563098:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:0.0:1773941613.627028:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1773941613.627035:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941613.627058:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941613.627061:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941613.627062:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941613.627063:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1773941613.627066:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:0.0:1773941613.627084:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:0.0:1773941613.627087:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:0.0:1773941613.627089:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1773941613.627090:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:0.0:1773941613.627097:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10837b80. 00000020:00000040:0.0:1773941613.627101:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941613.627105:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941613.627108:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941613.627112:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941613.627118:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065c5a00. 00000100:00000001:0.0:1773941613.627124:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941613.627127:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1773941613.627130:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1773941613.627137:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d21ec1400. 00010000:00000001:0.0:1773941613.627141:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1773941613.627144:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d00031600. 00000020:00000001:0.0:1773941613.627148:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:0.0:1773941613.627150:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d00031600 with handle 0x8e86767622e2ae1 to hash 00000020:00000001:0.0:1773941613.627151:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:0.0:1773941613.627152:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330090853888 : -113743618697728 : ffff988d00031600) 00010000:00000001:0.0:1773941613.627154:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330090853888 : -113743618697728 : ffff988d00031600) 00010000:00000001:0.0:1773941613.627158:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941613.627160:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941613.627163:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ae1 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941613.627177:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ae1 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941613.627187:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ae1 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941613.627191:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1773941613.627193:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941613.627202:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb6840. 00000100:00000001:0.0:1773941613.627203:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330139633728 : -113743569917888 : ffff988d02eb6840) 00000100:00000001:0.0:1773941613.627206:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941613.627207:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367703936 : -113743341847680 : ffff988d10837b80) 00000100:00000001:0.0:1773941613.627211:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941613.627214:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941613.627216:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941613.627217:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941613.627226:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10837b80 x1860112291499136/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941613.627233:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941613.627234:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941613.627237:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941613.627239:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941613.627241:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941613.627244:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10837b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291499136:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941613.627247:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941613.627251:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941613.627253:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941613.627255:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941613.627258:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d02ce7000. 02000000:00000001:0.0:1773941613.627259:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941613.627265:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941613.627268:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0014baa8. 00000400:00000010:0.0:1773941613.627273:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d108336e8. 00000100:00000200:0.0:1773941613.627279:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291499136, portal 25 00000100:00000001:0.0:1773941613.627280:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941613.627281:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367703936 : -113743341847680 : ffff988d10837b80) 00000100:00000040:0.0:1773941613.627285:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10837b80 x1860112291499136/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941629 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941613.627290:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941613.627292:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941613.627294:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d108333b8. 00000100:00000200:0.0:1773941613.627299:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291499136, offset 0 00000400:00000200:0.0:1773941613.627304:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941613.627311:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941613.627317:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941613.627321:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941613.627324:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941613.627328:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941613.627330:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941613.627332:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[176, 177] 00000400:00000200:0.0:1773941613.627335:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:0.0:1773941613.627338:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [288:288:256:4294967295] 192.168.204.114@tcp LPNI seq info [178:178:8:4294967295] 00000400:00000200:0.0:1773941613.627341:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941613.627348:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:0.0:1773941613.627353:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:0.0:1773941613.627356:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d113e9700. 00000800:00000200:0.0:1773941613.627362:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:0.0:1773941613.627368:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:0.0:1773941613.627371:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d113e9700 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1773941613.627386:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941613.627390:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941613.627392:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4295005223 00000100:00000001:0.0:1773941613.627398:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4295004009 00000100:00000001:0.0:1773941613.627402:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4295005223 00000100:00000001:0.0:1773941613.627404:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4295004009 00000100:00000001:0.0:1773941613.627406:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941613.627408:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941613.627410:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941613.627411:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1773941613.627413:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6840 going to sleep for 11 seconds 00000100:00000001:0.0:1773941613.627416:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941613.627424:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941613.627546:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941613.627552:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d113e9700. 00000400:00000200:1.0:1773941613.627559:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941613.627566:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941613.627580:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d108333b8 00000400:00000010:1.0:1773941613.627583:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d108333b8. 00000100:00000001:1.0:1773941613.627587:0:2394:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941613.627593:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10837b80 x1860112291499136/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941629 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941613.627604:0:2394:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941613.627608:0:2394:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d10837b80 x1860112291499136/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941629 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941613.627616:0:2394:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941613.627618:0:2394:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:1.0:1773941613.628731:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941613.628743:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941613.628745:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941613.628747:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941613.628751:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941613.628755:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941613.628760:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941613.628762:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941613.628765:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941613.628768:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324608c80 00000400:00000200:1.0:1773941613.628772:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941613.628776:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0x1259 [1] + 224 00000800:00000001:1.0:1773941613.628782:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941613.628789:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941613.628791:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941613.628793:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:1.0:1773941613.628797:0:2393:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941613.628800:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10837b80 x1860112291499136/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941629 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941613.628808:0:2393:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d10837b80 x1860112291499136/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941629 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941613.628821:0:2393:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1773941613.628833:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941613.628836:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941613.628846:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941613.628852:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941613.628853:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941613.628855:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0014baa8. 00000400:00000200:0.0:1773941613.628859:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d108336e8 00000400:00000010:0.0:1773941613.628860:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d108336e8. 00000100:00000001:0.0:1773941613.628862:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941613.628866:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10837b80 x1860112291499136/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941629 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941613.628874:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10837b80 x1860112291499136/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941629 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941613.628879:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941613.628881:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941613.628883:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941613.628885:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941613.628887:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941613.628888:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941613.628890:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941613.628895:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=114 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941613.628897:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941613.628900:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:0.0:1773941613.628903:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:0.0:1773941613.628906:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941613.628908:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:0.0:1773941613.628910:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:0.0:1773941613.628914:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:0.0:1773941613.628917:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:0.0:1773941613.628920:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:0.0:1773941613.628921:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:0.0:1773941613.628923:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:0.0:1773941613.628924:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:0.0:1773941613.628927:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d10837b80 x1860112291499136/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941629 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941613.628938:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:0.0:1773941613.628939:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:0.0:1773941613.628941:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941613.628943:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:0.0:1773941613.628945:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:0.0:1773941613.628954:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941613.628956:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941613.628957:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941613.628958:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:0.0:1773941613.628978:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941613.628979:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941613.628981:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1773941613.628983:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1773941613.628985:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1773941613.628988:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 335 00000400:00000200:0.0:1773941613.628991:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941613.628995:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:0.0:1773941613.628998:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941613.629001:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941613.629003:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941613.629005:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:0.0:1773941613.629007:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941613.629009:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:0.0:1773941613.629010:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941613.629012:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:0.0:1773941613.629014:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:0.0:1773941613.629015:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:0.0:1773941613.629017:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941613.629018:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941613.629019:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941613.629021:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:0.0:1773941613.629023:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:0.0:1773941613.629030:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:0.0:1773941613.629032:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941613.629034:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1773941613.629037:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941613.629038:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941613.629053:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:0.0:1773941613.629055:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1773941613.629058:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10836680. 00000020:00000040:0.0:1773941613.629060:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:0.0:1773941613.629062:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1773941613.629064:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941613.629066:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941613.629068:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941613.629069:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941613.629073:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d02ce7400. 00000100:00000001:0.0:1773941613.629075:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941613.629080:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d10836680 x1860112291499264/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1773941613.629086:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941613.629088:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941613.629091:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f5f5ddf5] to pc [ptlrpcd_rcv+-1] req@ffff988d10836680 x1860112291499264/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941613.629107:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941613.629109:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:0.0:1773941613.629111:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941613.629113:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941613.629115:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941613.629116:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1773941613.629118:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6840 going to sleep for 1 seconds 00000100:00000001:0.0:1773941613.629120:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941613.629122:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1773941613.629126:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10837b80 x1860112291499136/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941629 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941613.629132:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941613.629136:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941613.629139:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1773941613.629145:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000040:3.0:1773941613.629147:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10836680 x1860112291499264/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941613.629150:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941613.629155:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941613.629158:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941613.629163:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941613.629166:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941613.629168:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10837b80 x1860112291499136/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941629 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941613.629176:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10837b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291499136:192.168.204.114@tcp:101:ll_cfg_requeue.0 02000000:00000001:3.0:1773941613.629178:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1773941613.629180:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1773941613.629182:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941613.629184:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:3.0:1773941613.629185:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941613.629188:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1773941613.629191:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00100000:3.0:1773941613.629192:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10836680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291499264:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941613.629198:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000040:0.0:1773941613.629198:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10837b80 x1860112291499136/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941629 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1773941613.629202:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941613.629204:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941613.629208:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941613.629212:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03522400. 00000100:00000001:0.0:1773941613.629213:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941613.629217:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1773941613.629218:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb6840. 00000100:00000001:0.0:1773941613.629222:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000400:00000200:3.0:1773941613.629223:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941613.629224:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000010:3.0:1773941613.629226:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00010000:00000001:0.0:1773941613.629228:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1773941613.629232:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000400:00000010:3.0:1773941613.629233:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00000020:00000001:0.0:1773941613.629234:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941613.629237:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00031600 refcount=4 00000100:00000200:3.0:1773941613.629238:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291499264, portal 25 00000020:00000001:0.0:1773941613.629240:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090853888 : -113743618697728 : ffff988d00031600) 00000100:00000001:3.0:1773941613.629242:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00000001:0.0:1773941613.629244:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090853888 : -113743618697728 : ffff988d00031600) 00000100:00000001:3.0:1773941613.629245:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367698560 : -113743341853056 : ffff988d10836680) 00010000:00010000:0.0:1773941613.629249:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ae1 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:3.0:1773941613.629254:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10836680 x1860112291499264/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941623 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1773941613.629268:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941613.629270:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941613.629275:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00010000:0.0:1773941613.629275:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ae1 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:3.0:1773941613.629278:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac990. 00000100:00000200:3.0:1773941613.629280:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291499264, offset 0 00000400:00000200:3.0:1773941613.629287:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941613.629294:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:0.0:1773941613.629295:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1773941613.629298:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ae1 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1773941613.629302:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941613.629303:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1773941613.629307:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ae1 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941613.629310:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:0.0:1773941613.629314:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941613.629318:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941613.629320:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:0.0:1773941613.629322:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:3.0:1773941613.629326:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00010000:0.0:1773941613.629326:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ae1 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941613.629331:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941613.629337:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00010000:0.0:1773941613.629338:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d00031600 already unused, calling callback (ffffffffc12b4c20) 00000400:00000200:3.0:1773941613.629341:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[178, 177] 10000000:00000001:0.0:1773941613.629343:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1773941613.629347:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ae1 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941613.629348:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:3.0:1773941613.629355:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [289:289:256:4294967295] 192.168.204.124@tcp LPNI seq info [179:179:8:4294967295] 00010000:00000001:0.0:1773941613.629364:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1773941613.629365:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1773941613.629366:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941613.629369:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00031600 refcount=5 00000020:00000001:0.0:1773941613.629374:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090853888 : -113743618697728 : ffff988d00031600) 00000400:00000200:3.0:1773941613.629378:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00010000:00000001:0.0:1773941613.629379:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090853888 : -113743618697728 : ffff988d00031600) 00010000:00000001:0.0:1773941613.629383:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1773941613.629387:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ae1 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941613.629391:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 10000000:00000001:0.0:1773941613.629399:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000800:00000200:3.0:1773941613.629401:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:3.0:1773941613.629405:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74f00. 10000000:00010000:0.0:1773941613.629409:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ae1 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000800:00000200:3.0:1773941613.629413:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 10000000:01000000:0.0:1773941613.629421:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000800:00000200:3.0:1773941613.629425:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 10000000:00000040:0.0:1773941613.629427:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1773941613.629430:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941613.629431:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74f00 type 1, nob 616 niov 1 nkiov 1 00010000:00010000:0.0:1773941613.629433:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1773941613.629436:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1773941613.629438:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1773941613.629439:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1773941613.629440:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941613.629442:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941613.629443:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1773941613.629445:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d00031600 with handle 0x8e86767622e2ae1 from hash 00000400:00000200:3.0:1773941613.629449:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:0.0:1773941613.629449:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1773941613.629452:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941613.629453:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4295005225 00010000:00000001:0.0:1773941613.629455:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941613.629458:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000100:00000001:3.0:1773941613.629460:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4295005223 00010000:00000001:0.0:1773941613.629460:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1773941613.629461:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1773941613.629466:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941613.629467:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4295005225 00010000:00000001:0.0:1773941613.629468:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941613.629471:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941613.629474:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4295005223 10000000:00000001:0.0:1773941613.629474:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941613.629479:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ae1 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941613.629483:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1773941613.629486:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941613.629487:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941613.629489:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:3.0:1773941613.629491:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941613.629492:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1773941613.629495:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000100:00000001:3.0:1773941613.629496:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1773941613.629497:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941613.629500:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941613.629500:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1773941613.629502:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ae1 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941613.629503:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941613.629507:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000800:00000200:1.0:1773941613.629510:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1773941613.629511:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000040:0.0:1773941613.629511:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d21ec1400 count: 0 00000100:00000001:3.0:1773941613.629514:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000800:00000010:1.0:1773941613.629517:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74f00. 00000100:00000001:3.0:1773941613.629518:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941613.629522:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941613.629525:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000001:0.0:1773941613.629529:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941613.629533:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000400:00000200:1.0:1773941613.629536:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:0.0:1773941613.629537:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000400:00000200:1.0:1773941613.629544:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac990 00000100:00000040:0.0:1773941613.629544:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10837b80 x1860112291499136/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941629 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000010:1.0:1773941613.629550:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac990. 00000100:00000001:0.0:1773941613.629556:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:1.0:1773941613.629558:0:2394:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:0.0:1773941613.629559:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1773941613.629563:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d02ce7000. 02000000:00000001:0.0:1773941613.629566:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000200:1.0:1773941613.629569:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10836680 x1860112291499264/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941623 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1773941613.629569:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941613.629572:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:0.0:1773941613.629576:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1773941613.629580:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065c5a00. 02000000:00000001:0.0:1773941613.629586:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1773941613.629588:0:2394:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1773941613.629588:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1773941613.629592:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10837b80. 00000100:00000001:0.0:1773941613.629596:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000040:1.0:1773941613.629597:0:2394:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10836680 x1860112291499264/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941623 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941613.629598:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1773941613.629601:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:0.0:1773941613.629606:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:0.0:1773941613.629611:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1773941613.629614:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d10872940. 00000100:00000001:1.0:1773941613.629616:0:2394:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1773941613.629618:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d065c5a00. 00000100:00000001:1.0:1773941613.629620:0:2394:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:0.0:1773941613.629623:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d065c5c00. 00000040:00000010:0.0:1773941613.629627:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d065c4200. 00000020:00000010:0.0:1773941613.629631:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d054a7000. 00000020:00000010:0.0:1773941613.629635:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d065c4a00. 80000000:00000010:0.0:1773941613.629638:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d065803c0. 00040000:00000010:0.0:1773941613.629642:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d065c5400. 20000000:00000010:0.0:1773941613.629645:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d065c5e00. 10000000:00000040:0.0:1773941613.629649:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:0.0:1773941613.629650:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:0.0:1773941613.629654:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d065c5c00. 00000040:00000010:0.0:1773941613.629657:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d065c4200. 00000020:00000010:0.0:1773941613.629664:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d054a7000. 00000020:00000010:0.0:1773941613.629668:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d065c4a00. 80000000:00000010:0.0:1773941613.629670:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d065803c0. 00040000:00000010:0.0:1773941613.629672:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d065c5400. 20000000:00000010:0.0:1773941613.629674:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d065c5e00. 00000020:00000010:0.0:1773941613.629676:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d065c5a00. 10000000:00000010:0.0:1773941613.629678:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d10872940. 10000000:01000000:0.0:1773941613.629680:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:0.0:1773941613.629682:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:0.0:1773941613.629683:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:0.0:1773941613.629685:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1773941613.629686:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1773941613.629687:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941613.629689:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:1.0:1773941613.633917:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941613.633923:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941613.633925:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941613.633926:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941613.633930:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941613.633934:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:1.0:1773941613.633937:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941613.633940:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941613.633942:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941613.633945:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324608d00 00000400:00000200:1.0:1773941613.633947:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941613.633950:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0x1269 [1] + 0 00000800:00000001:1.0:1773941613.633954:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941613.633990:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941613.633992:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941613.633996:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:1.0:1773941613.633999:0:2393:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941613.634008:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10836680 x1860112291499264/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941623 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941613.634017:0:2393:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d10836680 x1860112291499264/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941623 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941613.634028:0:2393:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1773941613.634033:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941613.634036:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941613.634144:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941613.634148:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941613.634152:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941613.634153:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941613.634156:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941613.634161:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941613.634163:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941613.634167:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941613.634173:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10836680 x1860112291499264/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941623 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941613.634200:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10836680 x1860112291499264/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941623 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941613.634208:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941613.634210:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941613.634213:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941613.634214:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941613.634216:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941613.634218:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941613.634220:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941613.634224:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=114 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941613.634228:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941613.634231:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941613.634233:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941613.634236:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941613.634238:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941613.634241:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941613.634245:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10836680 x1860112291499264/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941623 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941613.634254:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941613.634256:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941613.634258:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941613.634261:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941613.634265:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941613.634268:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941613.634269:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941613.634271:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941613.634273:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941613.634276:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941613.634278:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941613.634282:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941613 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941613.634286:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941613.634292:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941613.634301:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941613.634303:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941613.634305:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941613.634307:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941613.634311:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941613.634313:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941613.634315:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941613.634319:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941613.634322:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941613.634324:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941613.634509:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941613.634512:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941613.634516:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10836680 x1860112291499264/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941623 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941613.634523:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10836680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291499264:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941613.634528:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941613.634530:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941613.634532:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10836680 x1860112291499264/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941623 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941613.634534:0:14815:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:3.0:1773941613.634540:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941613.634542:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00080000:1.0:1773941613.634544:0:14815:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 02000000:00000010:3.0:1773941613.634545:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03522400. 02000000:00000001:3.0:1773941613.634550:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00100000:1.0:1773941613.634551:0:14815:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000020:00000001:3.0:1773941613.634553:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:1.0:1773941613.634555:0:14815:0:(obd_class.h:1215:obd_import_event()) Process entered 00000020:00000040:3.0:1773941613.634556:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 10000000:01000000:1.0:1773941613.634558:0:14815:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000020:00000001:3.0:1773941613.634559:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941613.634562:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d02ce7400. 00000001:00000001:1.0:1773941613.634562:0:14815:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941613.634566:0:14815:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:3.0:1773941613.634567:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941613.634571:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000001:00000001:1.0:1773941613.634572:0:14815:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941613.634573:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10836680. 00000100:00000001:3.0:1773941613.634579:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000001:00000001:1.0:1773941613.634579:0:14815:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:3.0:1773941613.634583:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:1.0:1773941613.634583:0:14815:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941613.634585:0:14815:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941613.634587:0:14815:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941613.634588:0:14815:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941613.634593:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1773941613.634593:0:14815:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:3.0:1773941613.634596:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941613.634597:0:14815:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:3.0:1773941613.634600:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941613.634601:0:14815:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:3.0:1773941613.634604:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941613.634605:0:14815:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941613.634606:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941613.634610:0:14815:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941613.634615:0:14815:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941613.634617:0:14815:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941613.634619:0:14815:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:1.0:1773941613.634635:0:14815:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941613.634637:0:14815:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941613.634639:0:14815:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941613.634642:0:14815:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:1.0:1773941613.634646:0:14815:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941613.634649:0:14815:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941613.634660:0:14815:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941613.634662:0:14815:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941613.634664:0:14815:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:0.1:1773941613.635992:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d21ec1400. 00010000:00000010:0.1:1773941613.635999:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d00031600. 00000020:00000001:1.0:1773941614.587177:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1773941614.587184:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941614.587188:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 10000000:00000001:0.0:1773941614.843088:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1773941614.843096:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941614.843100:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941614.843102:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941614.843103:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941614.843105:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1773941614.843107:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:0.0:1773941614.843109:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:0.0:1773941614.843111:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:0.0:1773941614.843113:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1773941614.843115:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:0.0:1773941614.843124:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10837b80. 00000020:00000040:0.0:1773941614.843129:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941614.843133:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941614.843136:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941614.843140:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941614.843146:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065c5a00. 00000100:00000001:0.0:1773941614.843152:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941614.843155:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1773941614.843158:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1773941614.843164:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d21ec1400. 00010000:00000001:0.0:1773941614.843169:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1773941614.843183:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d00031600. 00000020:00000001:0.0:1773941614.843187:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:0.0:1773941614.843189:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d00031600 with handle 0x8e86767622e2ae8 to hash 00000020:00000001:0.0:1773941614.843191:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:0.0:1773941614.843192:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330090853888 : -113743618697728 : ffff988d00031600) 00010000:00000001:0.0:1773941614.843194:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330090853888 : -113743618697728 : ffff988d00031600) 00010000:00000001:0.0:1773941614.843198:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941614.843200:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941614.843204:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ae8 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941614.843212:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ae8 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941614.843222:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ae8 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941614.843227:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1773941614.843229:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941614.843234:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb6840. 00000100:00000001:0.0:1773941614.843236:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330139633728 : -113743569917888 : ffff988d02eb6840) 00000100:00000001:0.0:1773941614.843238:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941614.843239:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367703936 : -113743341847680 : ffff988d10837b80) 00000100:00000001:0.0:1773941614.843243:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941614.843247:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941614.843249:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941614.843250:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941614.843260:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10837b80 x1860112291499392/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941614.843267:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941614.843269:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941614.843272:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941614.843274:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941614.843279:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941614.843282:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10837b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291499392:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941614.843286:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941614.843290:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941614.843292:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941614.843294:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941614.843297:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d02ce7000. 02000000:00000001:0.0:1773941614.843298:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941614.843305:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941614.843316:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0014baa8. 00000400:00000010:0.0:1773941614.843322:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d108336e8. 00000100:00000200:0.0:1773941614.843328:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291499392, portal 25 00000100:00000001:0.0:1773941614.843329:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941614.843331:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367703936 : -113743341847680 : ffff988d10837b80) 00000100:00000040:0.0:1773941614.843335:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10837b80 x1860112291499392/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941630 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941614.843341:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941614.843343:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941614.843345:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833b28. 00000100:00000200:0.0:1773941614.843347:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291499392, offset 0 00000400:00000200:0.0:1773941614.843351:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941614.843359:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941614.843364:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941614.843368:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941614.843372:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941614.843376:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941614.843378:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941614.843381:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[178, 179] 00000400:00000200:0.0:1773941614.843383:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:0.0:1773941614.843387:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [290:290:256:4294967295] 192.168.204.114@tcp LPNI seq info [180:180:8:4294967295] 00000400:00000200:0.0:1773941614.843390:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941614.843397:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:0.0:1773941614.843406:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:0.0:1773941614.843410:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d113e9c00. 00000800:00000200:0.0:1773941614.843416:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:0.0:1773941614.843423:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:0.0:1773941614.843426:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d113e9c00 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1773941614.843442:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941614.843445:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941614.843447:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4295006439 00000100:00000001:0.0:1773941614.843453:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4295005225 00000100:00000001:0.0:1773941614.843456:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4295006439 00000100:00000001:0.0:1773941614.843459:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4295005225 00000100:00000001:0.0:1773941614.843462:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941614.843464:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941614.843466:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941614.843468:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1773941614.843471:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6840 going to sleep for 11 seconds 00000100:00000001:0.0:1773941614.843474:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941614.843476:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941614.843583:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941614.843588:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d113e9c00. 00000400:00000200:1.0:1773941614.843593:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941614.843598:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941614.843603:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833b28 00000400:00000010:1.0:1773941614.843605:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833b28. 00000100:00000001:1.0:1773941614.843608:0:2394:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941614.843612:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10837b80 x1860112291499392/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941630 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941614.843620:0:2394:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941614.843623:0:2394:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d10837b80 x1860112291499392/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941630 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941614.843628:0:2394:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941614.843630:0:2394:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:1.0:1773941614.844909:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941614.844922:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941614.844924:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941614.844926:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941614.844930:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941614.844934:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941614.844939:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941614.844942:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941614.844945:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941614.844948:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324608d80 00000400:00000200:1.0:1773941614.844951:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941614.844955:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0x1271 [1] + 224 00000800:00000001:1.0:1773941614.844989:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941614.845002:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941614.845004:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941614.845008:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:1.0:1773941614.845011:0:2393:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941614.845015:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10837b80 x1860112291499392/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941630 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941614.845039:0:2393:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d10837b80 x1860112291499392/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941630 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941614.845052:0:2393:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1773941614.845059:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941614.845062:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941614.845092:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941614.845100:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941614.845102:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941614.845105:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0014baa8. 00000400:00000200:0.0:1773941614.845109:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d108336e8 00000400:00000010:0.0:1773941614.845111:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d108336e8. 00000100:00000001:0.0:1773941614.845114:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941614.845121:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10837b80 x1860112291499392/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941630 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941614.845132:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10837b80 x1860112291499392/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941630 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941614.845140:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941614.845142:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941614.845152:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941614.845155:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941614.845158:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941614.845161:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941614.845163:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941614.845169:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=115 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941614.845172:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941614.845174:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:0.0:1773941614.845178:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:0.0:1773941614.845180:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941614.845183:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:0.0:1773941614.845186:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:0.0:1773941614.845190:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:0.0:1773941614.845193:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:0.0:1773941614.845196:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:0.0:1773941614.845198:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:0.0:1773941614.845199:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:0.0:1773941614.845201:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:0.0:1773941614.845204:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d10837b80 x1860112291499392/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941630 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941614.845209:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:0.0:1773941614.845210:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:0.0:1773941614.845211:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941614.845213:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:0.0:1773941614.845215:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:0.0:1773941614.845216:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941614.845217:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941614.845218:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941614.845219:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:0.0:1773941614.845220:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941614.845221:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941614.845222:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1773941614.845224:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1773941614.845226:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1773941614.845232:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 336 00000400:00000200:0.0:1773941614.845236:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941614.845239:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:0.0:1773941614.845243:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941614.845245:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941614.845248:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941614.845249:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:0.0:1773941614.845252:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941614.845253:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:0.0:1773941614.845255:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941614.845257:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:0.0:1773941614.845258:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:0.0:1773941614.845260:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:0.0:1773941614.845261:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941614.845263:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941614.845264:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941614.845265:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:0.0:1773941614.845268:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:0.0:1773941614.845270:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:0.0:1773941614.845272:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941614.845273:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1773941614.845275:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941614.845277:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941614.845279:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:0.0:1773941614.845280:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1773941614.845283:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10835f80. 00000020:00000040:0.0:1773941614.845285:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:0.0:1773941614.845287:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1773941614.845289:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941614.845290:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941614.845292:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941614.845293:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941614.845299:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d02ce6000. 00000100:00000001:0.0:1773941614.845302:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941614.845307:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d10835f80 x1860112291499520/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1773941614.845321:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941614.845324:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941614.845327:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000082649650] to pc [ptlrpcd_rcv+-1] req@ffff988d10835f80 x1860112291499520/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941614.845341:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941614.845343:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:0.0:1773941614.845345:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941614.845347:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941614.845348:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941614.845350:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1773941614.845351:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6840 going to sleep for 1 seconds 00000100:00000001:0.0:1773941614.845354:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941614.845355:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1773941614.845359:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10837b80 x1860112291499392/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941630 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941614.845365:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941614.845366:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941614.845369:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941614.845370:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941614.845372:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941614.845374:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1773941614.845375:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1773941614.845383:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10837b80 x1860112291499392/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941630 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941614.845384:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10835f80 x1860112291499520/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941614.845402:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10837b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291499392:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941614.845408:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941614.845411:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941614.845415:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1773941614.845416:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1773941614.845418:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941614.845419:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:3.0:1773941614.845426:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941614.845426:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1773941614.845429:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00100000:3.0:1773941614.845434:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10835f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291499520:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000040:0.0:1773941614.845442:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10837b80 x1860112291499392/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941630 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941614.845443:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941614.845448:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941614.845451:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941614.845452:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941614.845457:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000010:0.0:1773941614.845458:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb6840. 00000100:00000001:0.0:1773941614.845462:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1773941614.845463:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000010:3.0:1773941614.845465:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03522400. 00010000:00000001:0.0:1773941614.845466:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1773941614.845469:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 02000000:00000001:3.0:1773941614.845470:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941614.845472:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941614.845475:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00031600 refcount=4 00000400:00000200:3.0:1773941614.845476:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000020:00000001:0.0:1773941614.845479:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090853888 : -113743618697728 : ffff988d00031600) 00000400:00000010:3.0:1773941614.845481:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00010000:00000001:0.0:1773941614.845482:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090853888 : -113743618697728 : ffff988d00031600) 00010000:00010000:0.0:1773941614.845487:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ae8 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:3.0:1773941614.845488:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00000100:00000200:3.0:1773941614.845493:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291499520, portal 25 00000100:00000001:3.0:1773941614.845498:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00010000:00000001:0.0:1773941614.845499:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941614.845500:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367696768 : -113743341854848 : ffff988d10835f80) 00010000:00010000:0.0:1773941614.845504:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ae8 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:3.0:1773941614.845508:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10835f80 x1860112291499520/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941624 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1773941614.845517:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000100:00000001:3.0:1773941614.845522:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00010000:0.0:1773941614.845522:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ae8 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:3.0:1773941614.845528:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:3.0:1773941614.845532:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac198. 00010000:00000001:0.0:1773941614.845533:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000200:3.0:1773941614.845536:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291499520, offset 0 00010000:00000001:0.0:1773941614.845536:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1773941614.845539:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ae8 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1773941614.845544:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941614.845547:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941614.845550:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00000001:0.0:1773941614.845551:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1773941614.845554:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ae8 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941614.845556:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941614.845559:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941614.845565:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941614.845569:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941614.845573:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00010000:0.0:1773941614.845575:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d00031600 already unused, calling callback (ffffffffc12b4c20) 00000400:00000200:3.0:1773941614.845576:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941614.845581:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[180, 179] 10000000:00000001:0.0:1773941614.845581:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:3.0:1773941614.845586:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 10000000:00010000:0.0:1773941614.845587:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ae8 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941614.845589:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [291:291:256:4294967295] 192.168.204.124@tcp LPNI seq info [181:181:8:4294967295] 00000400:00000200:3.0:1773941614.845598:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:3.0:1773941614.845607:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00010000:00000001:0.0:1773941614.845607:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1773941614.845611:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000800:00000200:3.0:1773941614.845614:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.124@tcp 00000020:00000001:0.0:1773941614.845616:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000010:3.0:1773941614.845618:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74500. 00000020:00000040:0.0:1773941614.845620:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00031600 refcount=5 00000800:00000200:3.0:1773941614.845625:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000020:00000001:0.0:1773941614.845626:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090853888 : -113743618697728 : ffff988d00031600) 00010000:00000001:0.0:1773941614.845631:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090853888 : -113743618697728 : ffff988d00031600) 00000800:00000200:3.0:1773941614.845633:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:3.0:1773941614.845638:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74500 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:0.0:1773941614.845638:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1773941614.845644:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ae8 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941614.845651:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941614.845654:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000400:00000200:3.0:1773941614.845656:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:3.0:1773941614.845659:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4295006441 10000000:00010000:0.0:1773941614.845659:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ae8 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941614.845664:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4295006439 00000100:00000001:3.0:1773941614.845669:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4295006441 10000000:01000000:0.0:1773941614.845671:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000100:00000001:3.0:1773941614.845673:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4295006439 10000000:00000040:0.0:1773941614.845677:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000100:00000001:3.0:1773941614.845678:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941614.845682:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941614.845682:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941614.845686:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941614.845689:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:0.0:1773941614.845690:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1773941614.845694:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000100:00000001:3.0:1773941614.845695:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941614.845698:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00010000:00000001:0.0:1773941614.845698:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1773941614.845702:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000100:00000001:3.0:1773941614.845703:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00010000:00000001:0.0:1773941614.845706:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941614.845708:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941614.845711:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000020:00000040:0.0:1773941614.845714:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d00031600 with handle 0x8e86767622e2ae8 from hash 00000100:00000001:3.0:1773941614.845715:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941614.845720:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941614.845721:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1773941614.845724:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941614.845726:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000200:1.0:1773941614.845731:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00000001:0.0:1773941614.845735:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000800:00000010:1.0:1773941614.845737:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74500. 00010000:00000001:0.0:1773941614.845739:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1773941614.845744:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000400:00000200:1.0:1773941614.845745:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000001:0.0:1773941614.845748:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941614.845753:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:1.0:1773941614.845755:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00010000:00000001:0.0:1773941614.845756:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941614.845761:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941614.845764:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac198 00010000:00010000:0.0:1773941614.845766:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ae8 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:1.0:1773941614.845767:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac198. 00000100:00000001:1.0:1773941614.845773:0:2394:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941614.845779:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10835f80 x1860112291499520/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941624 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1773941614.845780:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941614.845784:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941614.845787:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1773941614.845791:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00000100:00000001:1.0:1773941614.845792:0:2394:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00010000:00000001:0.0:1773941614.845797:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000040:1.0:1773941614.845798:0:2394:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10835f80 x1860112291499520/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941624 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1773941614.845802:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941614.845806:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:1.0:1773941614.845807:0:2394:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941614.845810:0:2394:0:(events.c:68:request_out_callback()) Process leaving 00010000:00010000:0.0:1773941614.845811:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2ae8 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1773941614.845829:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d21ec1400 count: 0 00010000:00000001:0.0:1773941614.845840:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941614.845842:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941614.845845:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941614.845851:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10837b80 x1860112291499392/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941630 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941614.845859:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1773941614.845862:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1773941614.845865:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d02ce7000. 02000000:00000001:0.0:1773941614.845869:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1773941614.845871:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941614.845873:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:0.0:1773941614.845875:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1773941614.845878:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065c5a00. 02000000:00000001:0.0:1773941614.845882:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1773941614.845884:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1773941614.845892:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10837b80. 00000100:00000001:0.0:1773941614.845895:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941614.845896:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1773941614.845899:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:0.0:1773941614.845901:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:0.0:1773941614.845904:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1773941614.845908:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d10872940. 00000020:00000010:0.0:1773941614.845911:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d065c5a00. 00000020:00000010:0.0:1773941614.845916:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d065c5e00. 00000040:00000010:0.0:1773941614.845921:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d065c5400. 00000020:00000010:0.0:1773941614.845926:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d054a7000. 00000020:00000010:0.0:1773941614.845931:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d065c4a00. 80000000:00000010:0.0:1773941614.845935:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d065803c0. 00040000:00000010:0.0:1773941614.845939:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d065c4200. 20000000:00000010:0.0:1773941614.845944:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d065c5c00. 10000000:00000040:0.0:1773941614.845948:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:0.0:1773941614.845950:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:0.0:1773941614.845955:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d065c5e00. 00000040:00000010:0.0:1773941614.845959:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d065c5400. 00000020:00000010:0.0:1773941614.846000:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d054a7000. 00000020:00000010:0.0:1773941614.846005:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d065c4a00. 80000000:00000010:0.0:1773941614.846009:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d065803c0. 00040000:00000010:0.0:1773941614.846012:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d065c4200. 20000000:00000010:0.0:1773941614.846032:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d065c5c00. 00000020:00000010:0.0:1773941614.846038:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d065c5a00. 10000000:00000010:0.0:1773941614.846041:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d10872940. 10000000:01000000:0.0:1773941614.846044:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:0.0:1773941614.846048:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:0.0:1773941614.846051:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:0.0:1773941614.846053:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1773941614.846056:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1773941614.846058:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941614.846061:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:1.0:1773941614.846707:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941614.846714:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941614.846716:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941614.846718:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941614.846721:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941614.846724:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:1.0:1773941614.846729:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941614.846732:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941614.846741:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941614.846743:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324608e00 00000400:00000200:1.0:1773941614.846746:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941614.846749:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0x1281 [1] + 0 00000800:00000001:1.0:1773941614.846753:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941614.846757:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941614.846759:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941614.846762:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:1.0:1773941614.846764:0:2393:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941614.846768:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10835f80 x1860112291499520/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941624 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941614.846776:0:2393:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d10835f80 x1860112291499520/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941624 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941614.846790:0:2393:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1773941614.846794:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941614.846796:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941614.846812:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941614.846814:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941614.846817:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941614.846818:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941614.846819:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941614.846823:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941614.846824:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941614.846826:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941614.846830:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10835f80 x1860112291499520/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941624 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941614.846837:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10835f80 x1860112291499520/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941624 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941614.846842:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941614.846843:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941614.846844:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941614.846846:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941614.846847:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941614.846848:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941614.846849:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941614.846851:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=115 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941614.846854:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941614.846855:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941614.846856:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941614.846858:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941614.846860:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941614.846861:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941614.846864:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10835f80 x1860112291499520/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941624 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941614.846868:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941614.846870:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941614.846871:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941614.846872:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941614.846881:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941614.846882:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941614.846883:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941614.846885:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941614.846886:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941614.846888:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941614.846889:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941614.846892:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941614 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941614.846894:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941614.846898:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941614.846900:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941614.846901:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941614.846903:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941614.846904:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941614.846906:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941614.846908:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941614.846910:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941614.846912:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941614.846914:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941614.846916:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941614.847166:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941614.847170:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941614.847174:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10835f80 x1860112291499520/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941624 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941614.847181:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10835f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291499520:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941614.847186:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941614.847186:0:14816:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:3.0:1773941614.847190:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941614.847193:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10835f80 x1860112291499520/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941624 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1773941614.847199:0:14816:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00000001:3.0:1773941614.847200:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00100000:1.0:1773941614.847205:0:14816:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000001:3.0:1773941614.847207:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:1.0:1773941614.847209:0:14816:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000010:3.0:1773941614.847210:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03522400. 10000000:01000000:1.0:1773941614.847214:0:14816:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000001:3.0:1773941614.847215:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941614.847218:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941614.847220:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000001:00000001:1.0:1773941614.847220:0:14816:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:3.0:1773941614.847224:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941614.847225:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d02ce6000. 00000001:00000001:1.0:1773941614.847225:0:14816:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000001:3.0:1773941614.847229:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941614.847230:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000001:00000001:1.0:1773941614.847231:0:14816:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941614.847232:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10835f80. 00000100:00000001:3.0:1773941614.847238:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000001:00000001:1.0:1773941614.847238:0:14816:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941614.847241:0:14816:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:3.0:1773941614.847242:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941614.847245:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:1.0:1773941614.847245:0:14816:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941614.847250:0:14816:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941614.847252:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941614.847253:0:14816:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941614.847254:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941614.847257:0:14816:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:3.0:1773941614.847258:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941614.847261:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941614.847262:0:14816:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941614.847265:0:14816:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941614.847267:0:14816:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941614.847275:0:14816:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941614.847276:0:14816:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941614.847278:0:14816:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941614.847279:0:14816:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:1.0:1773941614.847290:0:14816:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941614.847291:0:14816:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941614.847292:0:14816:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941614.847295:0:14816:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:1.0:1773941614.847297:0:14816:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941614.847299:0:14816:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941614.847301:0:14816:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941614.847303:0:14816:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941614.847304:0:14816:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:0.1:1773941614.851007:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d21ec1400. 00010000:00000010:0.1:1773941614.851014:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d00031600. 00000020:00000001:2.0:1773941615.611215:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941615.611224:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941615.611233:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941616.123138:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4295006441, at 4295006464 00000100:00000010:1.0:1773941616.123159:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580420. 00000100:00000040:0.0:1773941616.123162:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4295006441, at 4295006464 00000100:00000010:0.0:1773941616.123199:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580b40. 10000000:00000001:0.0:1773941616.507098:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1773941616.507105:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941616.507108:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941616.507111:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941616.507112:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941616.507113:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1773941616.507117:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:0.0:1773941616.507118:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:0.0:1773941616.507121:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:0.0:1773941616.507123:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1773941616.507125:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:0.0:1773941616.507134:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10837b80. 00000020:00000040:0.0:1773941616.507139:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941616.507143:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941616.507146:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941616.507150:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941616.507156:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065c5a00. 00000100:00000001:0.0:1773941616.507163:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941616.507166:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1773941616.507169:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1773941616.507175:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d21ec1400. 00010000:00000001:0.0:1773941616.507180:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1773941616.507183:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d00031600. 00000020:00000001:0.0:1773941616.507186:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:0.0:1773941616.507188:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d00031600 with handle 0x8e86767622e2aef to hash 00000020:00000001:0.0:1773941616.507190:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:0.0:1773941616.507191:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330090853888 : -113743618697728 : ffff988d00031600) 00010000:00000001:0.0:1773941616.507193:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330090853888 : -113743618697728 : ffff988d00031600) 00010000:00000001:0.0:1773941616.507196:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941616.507199:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941616.507202:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2aef lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941616.507211:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2aef lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941616.507234:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2aef lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941616.507250:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1773941616.507251:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941616.507256:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb6840. 00000100:00000001:0.0:1773941616.507258:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330139633728 : -113743569917888 : ffff988d02eb6840) 00000100:00000001:0.0:1773941616.507261:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941616.507262:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367703936 : -113743341847680 : ffff988d10837b80) 00000100:00000001:0.0:1773941616.507266:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941616.507269:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941616.507271:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941616.507273:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941616.507282:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10837b80 x1860112291499648/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941616.507289:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941616.507292:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941616.507295:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941616.507298:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941616.507300:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941616.507303:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10837b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291499648:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941616.507306:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941616.507310:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941616.507312:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941616.507314:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941616.507317:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d02ce7000. 02000000:00000001:0.0:1773941616.507318:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941616.507336:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941616.507339:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0014baa8. 00000400:00000010:0.0:1773941616.507344:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d108336e8. 00000100:00000200:0.0:1773941616.507350:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291499648, portal 25 00000100:00000001:0.0:1773941616.507352:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941616.507357:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367703936 : -113743341847680 : ffff988d10837b80) 00000100:00000040:0.0:1773941616.507361:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10837b80 x1860112291499648/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941632 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941616.507368:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941616.507370:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941616.507372:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833220. 00000100:00000200:0.0:1773941616.507375:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291499648, offset 0 00000400:00000200:0.0:1773941616.507379:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941616.507386:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941616.507392:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941616.507397:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941616.507400:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941616.507403:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941616.507406:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941616.507408:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[180, 181] 00000400:00000200:0.0:1773941616.507411:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:0.0:1773941616.507419:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [292:292:256:4294967295] 192.168.204.114@tcp LPNI seq info [182:182:8:4294967295] 00000400:00000200:0.0:1773941616.507423:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941616.507432:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:0.0:1773941616.507438:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:0.0:1773941616.507442:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d113e9100. 00000800:00000200:0.0:1773941616.507449:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:0.0:1773941616.507455:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:0.0:1773941616.507458:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d113e9100 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1773941616.507473:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941616.507476:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941616.507478:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4295008103 00000100:00000010:0.0:1773941616.507482:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580b40. 00000100:00000001:0.0:1773941616.507487:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1773941616.507493:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4295008103 00000100:00000010:0.0:1773941616.507501:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d065803c0. 00000100:00000001:0.0:1773941616.507504:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1773941616.507507:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941616.507509:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941616.507511:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941616.507512:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1773941616.507514:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6840 going to sleep for 11 seconds 00000100:00000001:0.0:1773941616.507516:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941616.507519:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941616.507596:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941616.507601:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d113e9100. 00000400:00000200:1.0:1773941616.507605:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941616.507610:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941616.507614:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833220 00000400:00000010:1.0:1773941616.507616:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833220. 00000100:00000001:1.0:1773941616.507620:0:2394:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941616.507624:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10837b80 x1860112291499648/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941632 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941616.507632:0:2394:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941616.507635:0:2394:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d10837b80 x1860112291499648/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941632 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941616.507640:0:2394:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941616.507641:0:2394:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:1.0:1773941616.508977:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941616.509000:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941616.509002:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941616.509005:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941616.509009:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941616.509019:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941616.509024:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941616.509027:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941616.509030:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941616.509033:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324608e80 00000400:00000200:1.0:1773941616.509036:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941616.509039:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0x1289 [1] + 224 00000800:00000001:1.0:1773941616.509044:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941616.509050:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941616.509051:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941616.509054:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:1.0:1773941616.509057:0:2393:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941616.509060:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10837b80 x1860112291499648/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941632 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941616.509068:0:2393:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d10837b80 x1860112291499648/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941632 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941616.509086:0:2393:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1773941616.509094:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941616.509096:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941616.509126:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941616.509133:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941616.509135:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941616.509137:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0014baa8. 00000400:00000200:0.0:1773941616.509140:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d108336e8 00000400:00000010:0.0:1773941616.509142:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d108336e8. 00000100:00000001:0.0:1773941616.509144:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941616.509149:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10837b80 x1860112291499648/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941632 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941616.509158:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10837b80 x1860112291499648/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941632 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941616.509163:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941616.509165:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941616.509166:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941616.509168:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941616.509171:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941616.509172:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941616.509174:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941616.509179:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=117 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941616.509182:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941616.509184:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:0.0:1773941616.509187:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:0.0:1773941616.509189:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941616.509192:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:0.0:1773941616.509194:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:0.0:1773941616.509204:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:0.0:1773941616.509209:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:0.0:1773941616.509214:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:0.0:1773941616.509217:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:0.0:1773941616.509219:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:0.0:1773941616.509221:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:0.0:1773941616.509226:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d10837b80 x1860112291499648/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941632 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941616.509234:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:0.0:1773941616.509236:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:0.0:1773941616.509238:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941616.509240:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:0.0:1773941616.509243:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:0.0:1773941616.509245:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941616.509247:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941616.509248:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941616.509249:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:0.0:1773941616.509250:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941616.509252:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941616.509254:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1773941616.509256:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1773941616.509259:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1773941616.509262:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 337 00000400:00000200:0.0:1773941616.509267:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941616.509271:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:0.0:1773941616.509276:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941616.509279:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941616.509282:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941616.509285:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:0.0:1773941616.509288:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941616.509290:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:0.0:1773941616.509292:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941616.509301:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:0.0:1773941616.509303:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:0.0:1773941616.509306:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:0.0:1773941616.509310:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941616.509312:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941616.509314:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941616.509316:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:0.0:1773941616.509319:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:0.0:1773941616.509324:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:0.0:1773941616.509327:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941616.509329:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1773941616.509333:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941616.509335:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941616.509338:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:0.0:1773941616.509340:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1773941616.509344:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10835f80. 00000020:00000040:0.0:1773941616.509347:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:0.0:1773941616.509351:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1773941616.509353:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941616.509355:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941616.509357:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941616.509360:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941616.509364:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d02ce6800. 00000100:00000001:0.0:1773941616.509368:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941616.509374:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d10835f80 x1860112291499776/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1773941616.509382:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941616.509385:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941616.509390:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [0000000082649650] to pc [ptlrpcd_rcv+-1] req@ffff988d10835f80 x1860112291499776/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941616.509414:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941616.509417:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:0.0:1773941616.509424:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941616.509428:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941616.509431:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941616.509433:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1773941616.509436:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6840 going to sleep for 1 seconds 00000100:00000001:0.0:1773941616.509440:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941616.509443:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1773941616.509449:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10837b80 x1860112291499648/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941632 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941616.509458:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941616.509463:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941616.509463:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941616.509473:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941616.509474:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941616.509477:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941616.509478:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941616.509493:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10837b80 x1860112291499648/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941632 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:3.0:1773941616.509498:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10835f80 x1860112291499776/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941616.509543:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941616.509547:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941616.509556:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941616.509562:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1773941616.509562:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10837b80 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291499648:192.168.204.114@tcp:101:ll_cfg_requeue.0 02000000:00000001:3.0:1773941616.509570:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941616.509584:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10835f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291499776:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:0.0:1773941616.509586:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941616.509591:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1773941616.509596:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:3.0:1773941616.509598:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941616.509602:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941616.509605:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 02000000:00000001:3.0:1773941616.509606:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1773941616.509611:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000010:3.0:1773941616.509613:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03522400. 02000000:00000001:3.0:1773941616.509616:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941616.509623:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10837b80 x1860112291499648/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941632 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000400:00000200:3.0:1773941616.509628:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941616.509638:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00000100:00000001:0.0:1773941616.509641:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1773941616.509644:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb6840. 00000100:00000001:0.0:1773941616.509647:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1773941616.509648:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:0.0:1773941616.509650:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:0.0:1773941616.509652:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1773941616.509658:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941616.509661:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00031600 refcount=4 00000400:00000010:3.0:1773941616.509664:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00000020:00000001:0.0:1773941616.509665:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090853888 : -113743618697728 : ffff988d00031600) 00010000:00000001:0.0:1773941616.509673:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090853888 : -113743618697728 : ffff988d00031600) 00010000:00010000:0.0:1773941616.509682:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2aef lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:3.0:1773941616.509685:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291499776, portal 25 00000100:00000001:3.0:1773941616.509691:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941616.509694:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367696768 : -113743341854848 : ffff988d10835f80) 00010000:00000001:0.0:1773941616.509703:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000040:3.0:1773941616.509706:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10835f80 x1860112291499776/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941626 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:0.0:1773941616.509717:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2aef lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941616.509719:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941616.509724:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:3.0:1773941616.509731:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac770. 00000100:00000200:3.0:1773941616.509737:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291499776, offset 0 00010000:00000001:0.0:1773941616.509748:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000400:00000200:3.0:1773941616.509751:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00010000:0.0:1773941616.509767:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2aef lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941616.509788:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:0.0:1773941616.509797:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000200:3.0:1773941616.509798:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:0.0:1773941616.509804:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:3.0:1773941616.509807:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00010000:0.0:1773941616.509812:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2aef lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941616.509813:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941616.509819:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941616.509833:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941616.509848:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[182, 181] 00010000:00000001:0.0:1773941616.509850:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941616.509854:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941616.509855:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00010000:00000001:0.0:1773941616.509862:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:0.0:1773941616.509867:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2aef lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941616.509868:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [293:293:256:4294967295] 192.168.204.124@tcp LPNI seq info [183:183:8:4294967295] 00000400:00000200:3.0:1773941616.509896:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00010000:00010000:0.0:1773941616.509908:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d00031600 already unused, calling callback (ffffffffc12b4c20) 10000000:00000001:0.0:1773941616.509911:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1773941616.509917:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2aef lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941616.509926:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00010000:00000001:0.0:1773941616.509938:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00000001:0.0:1773941616.509941:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1773941616.509947:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000200:3.0:1773941616.509951:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.124@tcp 00000020:00000040:0.0:1773941616.509953:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00031600 refcount=5 00000800:00000010:3.0:1773941616.509960:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74400. 00000800:00000200:3.0:1773941616.509969:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000020:00000001:0.0:1773941616.510003:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090853888 : -113743618697728 : ffff988d00031600) 00010000:00000001:0.0:1773941616.510007:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090853888 : -113743618697728 : ffff988d00031600) 00010000:00000001:0.0:1773941616.510010:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00010000:0.0:1773941616.510013:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2aef lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1773941616.510018:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1773941616.510020:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2aef lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:0.0:1773941616.510025:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:0.0:1773941616.510027:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1773941616.510028:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941616.510031:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00000001:0.0:1773941616.510033:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1773941616.510036:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1773941616.510037:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:0.0:1773941616.510039:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941616.510041:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1773941616.510042:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d00031600 with handle 0x8e86767622e2aef from hash 00010000:00000001:0.0:1773941616.510044:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00010000:00000001:0.0:1773941616.510045:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941616.510046:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941616.510048:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00010000:00000001:0.0:1773941616.510049:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00000001:0.0:1773941616.510052:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00010000:00000001:0.0:1773941616.510055:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941616.510059:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000200:3.0:1773941616.510065:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00010000:00000001:0.0:1773941616.510065:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941616.510071:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941616.510082:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74400 type 1, nob 616 niov 1 nkiov 1 00010000:00010000:0.0:1773941616.510083:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2aef lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1773941616.510103:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941616.510109:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941616.510110:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1773941616.510111:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1773941616.510112:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941616.510114:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941616.510115:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:3.0:1773941616.510118:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941616.510119:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d00031600/0x8e86767622e2aef lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941616.510132:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:3.0:1773941616.510143:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4295008106 00010000:00000040:0.0:1773941616.510144:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d21ec1400 count: 0 00000100:00000001:3.0:1773941616.510160:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4295008103 00010000:00000001:0.0:1773941616.510164:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941616.510166:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941616.510169:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941616.510178:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10837b80 x1860112291499648/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941632 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941616.510179:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4295008106 00000800:00000200:1.0:1773941616.510183:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:3.0:1773941616.510191:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4295008103 00000800:00000010:1.0:1773941616.510192:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74400. 00000100:00000001:0.0:1773941616.510201:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1773941616.510205:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000200:1.0:1773941616.510206:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 02000000:00000001:0.0:1773941616.510207:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1773941616.510214:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d02ce7000. 00000100:00000001:3.0:1773941616.510216:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941616.510225:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1773941616.510227:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941616.510227:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000020:00000001:0.0:1773941616.510229:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941616.510236:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941616.510240:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941616.510241:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac770 00000020:00000001:0.0:1773941616.510241:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1773941616.510249:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065c5a00. 00000400:00000010:1.0:1773941616.510250:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac770. 00000100:00000001:3.0:1773941616.510257:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:0.0:1773941616.510257:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1773941616.510258:0:2394:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:0.0:1773941616.510263:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1773941616.510266:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000010:0.0:1773941616.510266:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10837b80. 00000100:00000200:1.0:1773941616.510269:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10835f80 x1860112291499776/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941626 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941616.510276:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941616.510279:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941616.510282:0:2394:0:(client.c:2842:__ptlrpc_req_put()) Process entered 10000000:00000001:0.0:1773941616.510282:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:0.0:1773941616.510286:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 00000100:00000040:1.0:1773941616.510287:0:2394:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10835f80 x1860112291499776/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941626 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:00000001:0.0:1773941616.510289:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1773941616.510294:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d10872940. 00000020:00000010:0.0:1773941616.510298:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d065c5a00. 00000020:00000010:0.0:1773941616.510303:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d065c5c00. 00000040:00000010:0.0:1773941616.510308:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d065c4200. 00000020:00000010:0.0:1773941616.510313:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d054a7000. 00000100:00000001:1.0:1773941616.510316:0:2394:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941616.510329:0:2394:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000010:0.0:1773941616.510338:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d065c4a00. 80000000:00000010:0.0:1773941616.510341:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d065805a0. 00040000:00000010:0.0:1773941616.510346:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d065c5400. 20000000:00000010:0.0:1773941616.510349:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d065c5e00. 10000000:00000040:0.0:1773941616.510354:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:0.0:1773941616.510356:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:0.0:1773941616.510360:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d065c5c00. 00000040:00000010:0.0:1773941616.510364:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d065c4200. 00000020:00000010:0.0:1773941616.510368:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d054a7000. 00000020:00000010:0.0:1773941616.510372:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d065c4a00. 00000100:00000001:3.0:1773941616.510377:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 80000000:00000010:0.0:1773941616.510377:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d065805a0. 00000100:00000001:3.0:1773941616.510383:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00040000:00000010:0.0:1773941616.510383:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d065c5400. 00000100:00000001:3.0:1773941616.510392:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 20000000:00000010:0.0:1773941616.510392:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d065c5e00. 00000100:00000001:3.0:1773941616.510398:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1773941616.510398:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d065c5a00. 10000000:00000010:0.0:1773941616.510413:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d10872940. 10000000:01000000:0.0:1773941616.510422:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:0.0:1773941616.510425:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:0.0:1773941616.510428:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:0.0:1773941616.510430:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1773941616.510431:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1773941616.510432:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941616.510435:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:1.0:1773941616.512018:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941616.512032:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941616.512035:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941616.512037:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941616.512043:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941616.512048:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:1.0:1773941616.512055:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941616.512059:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941616.512062:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941616.512066:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324608f00 00000400:00000200:1.0:1773941616.512070:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941616.512075:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0x1299 [1] + 0 00000800:00000001:1.0:1773941616.512082:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941616.512090:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941616.512092:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941616.512096:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:1.0:1773941616.512102:0:2393:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941616.512107:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10835f80 x1860112291499776/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941626 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941616.512117:0:2393:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d10835f80 x1860112291499776/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941626 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941616.512135:0:2393:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1773941616.512144:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941616.512147:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941616.512165:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941616.512169:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941616.512174:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941616.512175:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941616.512178:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941616.512183:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941616.512185:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941616.512188:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941616.512193:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10835f80 x1860112291499776/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941626 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941616.512203:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10835f80 x1860112291499776/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941626 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941616.512210:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941616.512211:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941616.512213:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941616.512216:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941616.512218:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941616.512220:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941616.512222:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941616.512228:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=117 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941616.512231:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=2 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941616.512233:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941616.512235:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941616.512247:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941616.512250:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941616.512252:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941616.512257:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10835f80 x1860112291499776/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941626 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941616.512263:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941616.512266:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941616.512268:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941616.512270:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941616.512277:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941616.512279:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941616.512280:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941616.512283:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941616.512286:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941616.512288:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941616.512290:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941616.512294:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941616 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941616.512297:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941616.512303:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941616.512305:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941616.512308:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941616.512310:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941616.512311:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941616.512315:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941616.512317:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941616.512320:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941616.512323:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941616.512326:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941616.512328:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941616.512595:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941616.512599:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941616.512603:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10835f80 x1860112291499776/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941626 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941616.512621:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10835f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291499776:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941616.512627:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941616.512629:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941616.512632:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10835f80 x1860112291499776/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941626 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941616.512637:0:14817:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:3.0:1773941616.512639:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941616.512643:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941616.512646:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03522400. 02000000:00000001:3.0:1773941616.512649:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941616.512651:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941616.512655:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00080000:1.0:1773941616.512655:0:14817:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000020:00000001:3.0:1773941616.512659:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00100000:1.0:1773941616.512661:0:14817:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000010:3.0:1773941616.512662:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d02ce6800. 00000100:00000001:1.0:1773941616.512668:0:14817:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:3.0:1773941616.512669:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 10000000:01000000:1.0:1773941616.512671:0:14817:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 02000000:00000001:3.0:1773941616.512672:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941616.512674:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10835f80. 00000100:00000001:3.0:1773941616.512678:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941616.512679:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941616.512683:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:1.0:1773941616.512685:0:14817:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:3.0:1773941616.512687:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941616.512691:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941616.512691:0:14817:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:3.0:1773941616.512697:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941616.512700:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941616.512700:0:14817:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941616.512705:0:14817:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941616.512706:0:14817:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941616.512707:0:14817:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941616.512709:0:14817:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941616.512711:0:14817:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:1.0:1773941616.512716:0:14817:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:1.0:1773941616.512719:0:14817:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941616.512721:0:14817:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941616.512723:0:14817:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941616.512726:0:14817:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941616.512728:0:14817:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941616.512729:0:14817:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941616.512731:0:14817:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:1.0:1773941616.512750:0:14817:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941616.512752:0:14817:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941616.512753:0:14817:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941616.512757:0:14817:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:1.0:1773941616.512760:0:14817:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941616.512763:0:14817:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941616.512764:0:14817:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941616.512767:0:14817:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941616.512768:0:14817:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:0.1:1773941616.516008:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d21ec1400. 00010000:00000010:0.1:1773941616.516016:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d00031600. 00000020:00000001:1.0:1773941616.635203:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1773941616.635212:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941616.635217:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1773941617.659254:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941617.659264:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941617.659269:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:1.0:1773941617.787124:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4295008106, at 4295008128 00000100:00000010:1.0:1773941617.787149:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d065803c0. 00000100:00000040:0.0:1773941617.787152:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4295008106, at 4295008128 00000100:00000010:0.0:1773941617.787170:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580b40. 00000100:00000040:1.0:1773941617.915204:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) c1c1ff21-5f68-4bf6-bdc4-258644b24465->MGS: level FULL/9 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1773941617.915222:0:36:0:(pinger.c:119:ptlrpc_ping()) Process entered 00000100:00000010:1.0:1773941617.915241:0:36:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d203c0000. 00000020:00000040:1.0:1773941617.915250:0:36:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:1.0:1773941617.915258:0:36:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1773941617.915289:0:36:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1773941617.915301:0:36:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:1.0:1773941617.915319:0:36:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff988d02cfda00. 00000100:00000001:1.0:1773941617.915333:0:36:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941617.915357:0:36:0:(pinger.c:133:ptlrpc_ping()) @@@ pinging c1c1ff21-5f68-4bf6-bdc4-258644b24465->MGS req@ffff988d203c0000 x1860112291499904/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1773941617.915382:0:36:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1773941617.915391:0:36:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941617.915405:0:36:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [000000001ea6e639] to pc [ptlrpcd_00_00+0] req@ffff988d203c0000 x1860112291499904/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941617.915463:0:36:0:(pinger.c:142:ptlrpc_ping()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:1.0:1773941617.915471:0:36:0:(pinger.c:302:ptlrpc_pinger_main()) next wakeup in 5 (345) 00000100:00000001:1.0:1773941617.915493:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1773941617.915499:0:2402:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00000001:1.0:1773941617.915504:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941617.915516:0:2402:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1773941617.915520:0:2402:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941617.915525:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941617.915529:0:2402:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1773941617.915533:0:2402:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:1.0:1773941617.915545:0:2402:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d203c0000 x1860112291499904/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941617.915549:0:2401:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941617.915560:0:2401:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941617.915568:0:2402:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1773941617.915576:0:2402:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941617.915586:0:2402:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1773941617.915592:0:2402:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1773941617.915599:0:2402:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1773941617.915608:0:2402:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d203c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2402:1860112291499904:192.168.204.114@tcp:400:kworker.0 00000100:00000001:1.0:1773941617.915618:0:2402:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1773941617.915628:0:2402:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1773941617.915633:0:2402:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1773941617.915639:0:2402:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:1.0:1773941617.915649:0:2402:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff988d181fc800. 02000000:00000001:1.0:1773941617.915653:0:2402:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941617.915681:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:1.0:1773941617.915691:0:2402:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d02d1c840. 00000400:00000010:1.0:1773941617.915706:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113de6e8. 00000100:00000200:1.0:1773941617.915721:0:2402:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112291499904, portal 25 00000100:00000001:1.0:1773941617.915727:0:2402:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1773941617.915730:0:2402:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330631454720 : -113743078096896 : ffff988d203c0000) 00000100:00000040:1.0:1773941617.915742:0:2402:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d203c0000 x1860112291499904/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941633 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941617.915758:0:2402:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:1.0:1773941617.915763:0:2402:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:1.0:1773941617.915767:0:2402:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d113dee58. 00000100:00000200:1.0:1773941617.915771:0:2402:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1860112291499904, offset 0 00000400:00000200:1.0:1773941617.915781:0:2402:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:1.0:1773941617.915795:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941617.915807:0:2402:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:1.0:1773941617.915815:0:2402:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:1.0:1773941617.915822:0:2402:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:1.0:1773941617.915828:0:2402:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941617.915832:0:2402:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:1.0:1773941617.915836:0:2402:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[182, 183] 00000400:00000200:1.0:1773941617.915842:0:2402:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:1.0:1773941617.915847:0:2402:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [294:294:256:4294967295] 192.168.204.114@tcp LPNI seq info [184:184:8:4294967295] 00000400:00000200:1.0:1773941617.915853:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941617.915866:0:2402:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:1.0:1773941617.915875:0:2402:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:1.0:1773941617.915882:0:2402:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d02cfd200. 00000800:00000200:1.0:1773941617.915892:0:2402:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:1.0:1773941617.915905:0:2402:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:1.0:1773941617.915913:0:2402:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d02cfd200 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:1.0:1773941617.915938:0:2402:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941617.915944:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:1.0:1773941617.916027:0:2402:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4295009511 00000100:00000010:1.0:1773941617.916039:0:2402:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3d80. 00000100:00000001:1.0:1773941617.916050:0:2402:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1773941617.916060:0:2402:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4295009512 00000100:00000010:1.0:1773941617.916067:0:2402:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d054f3e40. 00000100:00000001:1.0:1773941617.916072:0:2402:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1773941617.916078:0:2402:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1773941617.916082:0:2402:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941617.916087:0:2402:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941617.916092:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941617.916160:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941617.916168:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d02cfd200. 00000400:00000200:0.0:1773941617.916177:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941617.916187:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:0.0:1773941617.916196:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113dee58 00000400:00000010:0.0:1773941617.916200:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113dee58. 00000100:00000001:0.0:1773941617.916205:0:2394:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941617.916213:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d203c0000 x1860112291499904/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941633 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941617.916242:0:2394:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941617.916247:0:2394:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d203c0000 x1860112291499904/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941633 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941617.916258:0:2394:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941617.916260:0:2394:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:1.0:1773941617.918143:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941617.918161:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941617.918164:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941617.918168:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941617.918175:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941617.918183:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941617.918191:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941617.918196:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941617.918203:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941617.918209:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324608f80 00000400:00000200:1.0:1773941617.918214:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941617.918221:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0x12a1 [1] + 224 00000800:00000001:1.0:1773941617.918232:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941617.918244:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941617.918248:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941617.918254:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:1.0:1773941617.918261:0:2393:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941617.918267:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d203c0000 x1860112291499904/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941633 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941617.918281:0:2393:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff988d203c0000 x1860112291499904/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941633 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941617.918311:0:2393:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1773941617.918338:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941617.918345:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941617.918421:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941617.918427:0:2402:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941617.918437:0:2402:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941617.918439:0:2402:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941617.918444:0:2402:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d02d1c840. 00000400:00000200:0.0:1773941617.918451:0:2402:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d113de6e8 00000400:00000010:0.0:1773941617.918453:0:2402:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d113de6e8. 00000100:00000001:0.0:1773941617.918457:0:2402:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941617.918466:0:2402:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d203c0000 x1860112291499904/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941633 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941617.918481:0:2402:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d203c0000 x1860112291499904/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941633 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941617.918491:0:2402:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941617.918493:0:2402:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941617.918497:0:2402:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941617.918502:0:2402:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941617.918505:0:2402:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941617.918508:0:2402:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941617.918512:0:2402:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941617.918521:0:2402:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=118 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941617.918525:0:2402:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941617.918529:0:2402:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:0.0:1773941617.918532:0:2402:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941617.918536:0:2402:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:0.0:1773941617.918541:0:2402:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:0.0:1773941617.918548:0:2402:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:0.0:1773941617.918555:0:2402:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:0.0:1773941617.918576:0:2402:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:0.0:1773941617.918581:0:2402:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:0.0:1773941617.918584:0:2402:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:0.0:1773941617.918587:0:2402:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:0.0:1773941617.918594:0:2402:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d203c0000 x1860112291499904/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941633 ref 1 fl Rpc:RNQU/200/0 rc 0/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941617.918605:0:2402:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:0.0:1773941617.918607:0:2402:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:0.0:1773941617.918610:0:2402:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941617.918615:0:2402:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:0.0:1773941617.918618:0:2402:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:0.0:1773941617.918621:0:2402:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941617.918623:0:2402:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941617.918625:0:2402:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941617.918626:0:2402:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:0.0:1773941617.918628:0:2402:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941617.918630:0:2402:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941617.918635:0:2402:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1773941617.918638:0:2402:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1773941617.918643:0:2402:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1773941617.918647:0:2402:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 339 00000400:00000200:0.0:1773941617.918653:0:2402:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941617.918662:0:2402:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:0.0:1773941617.918668:0:2402:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941617.918673:0:2402:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941617.918676:0:2402:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941617.918679:0:2402:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:0.0:1773941617.918684:0:2402:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941617.918687:0:2402:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:0.0:1773941617.918691:0:2402:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941617.918695:0:2402:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:0.0:1773941617.918698:0:2402:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:0.0:1773941617.918701:0:2402:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:0.0:1773941617.918711:0:2402:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941617.918714:0:2402:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941617.918717:0:2402:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941617.918719:0:2402:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:0.0:1773941617.918724:0:2402:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:0.0:1773941617.918728:0:2402:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:0.0:1773941617.918731:0:2402:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941617.918735:0:2402:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1773941617.918739:0:2402:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941617.918743:0:2402:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941617.918746:0:2402:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:0.0:1773941617.918748:0:2402:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1773941617.918754:0:2402:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10837b80. 00000020:00000040:0.0:1773941617.918757:0:2402:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:0.0:1773941617.918762:0:2402:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1773941617.918764:0:2402:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941617.918768:0:2402:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941617.918771:0:2402:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941617.918774:0:2402:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941617.918780:0:2402:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d02ce5800. 00000100:00000001:0.0:1773941617.918786:0:2402:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941617.918793:0:2402:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d10837b80 x1860112291500032/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1773941617.918823:0:2402:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941617.918827:0:2402:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941617.918832:0:2402:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000d4002286] to pc [ptlrpcd_rcv+-1] req@ffff988d10837b80 x1860112291500032/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941617.918859:0:2402:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941617.918862:0:2402:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:0.0:1773941617.918865:0:2402:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:0.0:1773941617.918871:0:2402:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d203c0000 x1860112291499904/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941633 ref 1 fl Rpc:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941617.918897:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941617.918898:0:2402:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941617.918906:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941617.918907:0:2402:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941617.918913:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1773941617.918914:0:2402:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:3.0:1773941617.918929:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10837b80 x1860112291500032/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000040:0.0:1773941617.918934:0:2402:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d203c0000 x1860112291499904/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941633 ref 1 fl Interpret:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941617.919020:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941617.919023:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941617.919028:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941617.919031:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941617.919035:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941617.919068:0:2402:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d203c0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2402:1860112291499904:192.168.204.114@tcp:400:kworker.0 00000100:00100000:3.0:1773941617.919084:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10837b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291500032:192.168.204.114@tcp:250:ptlrpcd_00_01.0 00000100:00000001:0.0:1773941617.919088:0:2402:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941617.919093:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1773941617.919098:0:2402:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1773941617.919100:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941617.919107:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941617.919110:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000040:0.0:1773941617.919118:0:2402:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d203c0000 x1860112291499904/t0(0) o400->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941633 ref 1 fl Complete:RENQU/200/0 rc -107/-107 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000010:3.0:1773941617.919120:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03522400. 02000000:00000001:3.0:1773941617.919127:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941617.919141:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941617.919150:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00000100:00000001:0.0:1773941617.919162:0:2402:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000400:00000010:3.0:1773941617.919164:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 02000000:00000001:0.0:1773941617.919173:0:2402:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000200:3.0:1773941617.919174:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291500032, portal 25 00000100:00000001:3.0:1773941617.919180:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941617.919183:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367703936 : -113743341847680 : ffff988d10837b80) 02000000:00000010:0.0:1773941617.919185:0:2402:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff988d181fc800. 00000100:00000040:3.0:1773941617.919196:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10837b80 x1860112291500032/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941627 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1773941617.919198:0:2402:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1773941617.919205:0:2402:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941617.919211:0:2402:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:0.0:1773941617.919218:0:2402:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1773941617.919225:0:2402:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff988d02cfda00. 00000100:00000001:3.0:1773941617.919229:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1773941617.919232:0:2402:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1773941617.919238:0:2402:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000040:3.0:1773941617.919241:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000100:00000010:0.0:1773941617.919243:0:2402:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d203c0000. 00000400:00000010:3.0:1773941617.919253:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182accc0. 00000100:00000001:0.0:1773941617.919253:0:2402:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941617.919257:0:2402:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941617.919265:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000200:3.0:1773941617.919269:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291500032, offset 0 00000100:00000001:0.0:1773941617.919271:0:2402:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941617.919276:0:2402:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941617.919283:0:2402:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941617.919288:0:2402:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941617.919291:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941617.919305:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941617.919316:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941617.919324:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941617.919329:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941617.919335:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941617.919340:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941617.919347:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[184, 183] 00000400:00000200:3.0:1773941617.919355:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:3.0:1773941617.919360:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [295:295:256:4294967295] 192.168.204.124@tcp LPNI seq info [185:185:8:4294967295] 00000400:00000200:3.0:1773941617.919366:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:3.0:1773941617.919377:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:3.0:1773941617.919397:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:3.0:1773941617.919403:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74400. 00000800:00000200:3.0:1773941617.919410:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:3.0:1773941617.919419:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:3.0:1773941617.919424:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74400 type 1, nob 616 niov 1 nkiov 1 00000100:00000001:3.0:1773941617.919443:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941617.919449:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:3.0:1773941617.919452:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4295009515 00000100:00000001:3.0:1773941617.919461:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4295009511 00000100:00000001:3.0:1773941617.919466:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4295009515 00000100:00000001:3.0:1773941617.919472:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4295009512 00000100:00000001:3.0:1773941617.919479:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941617.919483:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941617.919487:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941617.919491:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941617.919497:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941617.919500:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1773941617.919503:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941617.919505:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941617.919507:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941617.919510:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:0.0:1773941617.919557:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941617.919563:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74400. 00000400:00000200:0.0:1773941617.919568:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941617.919575:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941617.919590:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182accc0 00000400:00000010:0.0:1773941617.919593:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182accc0. 00000100:00000001:0.0:1773941617.919597:0:2394:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941617.919603:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10837b80 x1860112291500032/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941627 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941617.919622:0:2394:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941617.919626:0:2394:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10837b80 x1860112291500032/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941627 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941617.919637:0:2394:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941617.919640:0:2394:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:1.0:1773941617.921593:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941617.921606:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941617.921610:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941617.921616:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941617.921624:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941617.921632:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:1.0:1773941617.921640:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941617.921646:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941617.921652:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941617.921659:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324609000 00000400:00000200:1.0:1773941617.921667:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941617.921674:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0x12b1 [1] + 0 00000800:00000001:1.0:1773941617.921684:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941617.921719:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941617.921725:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941617.921733:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:1.0:1773941617.921741:0:2393:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941617.921750:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10837b80 x1860112291500032/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941627 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941617.921765:0:2393:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d10837b80 x1860112291500032/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941627 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941617.921790:0:2393:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1773941617.921798:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941617.921803:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941617.921838:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941617.921844:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941617.921850:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941617.921864:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941617.921867:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941617.921874:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941617.921877:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941617.921881:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941617.921890:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10837b80 x1860112291500032/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941627 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941617.921905:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10837b80 x1860112291500032/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941627 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941617.921934:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941617.921936:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941617.921939:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941617.921942:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941617.921944:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941617.921947:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941617.921950:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941617.921955:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=118 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941617.922010:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941617.922014:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941617.922017:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941617.922021:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941617.922024:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941617.922028:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941617.922034:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10837b80 x1860112291500032/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941627 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941617.922044:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941617.922047:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941617.922049:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941617.922053:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941617.922061:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941617.922064:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941617.922066:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941617.922069:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941617.922072:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941617.922075:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941617.922078:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941617.922083:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941617 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941617.922087:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941617.922096:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941617.922099:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941617.922101:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941617.922104:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941617.922116:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941617.922121:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941617.922125:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941617.922128:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941617.922133:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941617.922138:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941617.922141:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941617.922544:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941617.922550:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941617.922559:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10837b80 x1860112291500032/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941627 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941617.922571:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10837b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291500032:192.168.204.114@tcp:250:ptlrpcd_00_01.0 00000100:00000001:3.0:1773941617.922579:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941617.922591:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941617.922596:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10837b80 x1860112291500032/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941627 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ptlrpcd_00_01.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941617.922605:0:14818:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00000001:3.0:1773941617.922606:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941617.922612:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941617.922617:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03522400. 02000000:00000001:3.0:1773941617.922622:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1773941617.922625:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941617.922629:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00080000:1.0:1773941617.922632:0:14818:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000020:00000001:3.0:1773941617.922634:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:3.0:1773941617.922641:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d02ce5800. 00000100:00100000:1.0:1773941617.922646:0:14818:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 02000000:00000001:3.0:1773941617.922647:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1773941617.922651:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941617.922657:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10837b80. 00000100:00000001:1.0:1773941617.922658:0:14818:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:3.0:1773941617.922662:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 10000000:01000000:1.0:1773941617.922666:0:14818:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000100:00000001:3.0:1773941617.922667:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941617.922675:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000001:00000001:1.0:1773941617.922679:0:14818:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:3.0:1773941617.922682:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941617.922687:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941617.922689:0:14818:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:3.0:1773941617.922702:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941617.922709:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941617.922716:0:14818:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:1.0:1773941617.922724:0:14818:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:1.0:1773941617.922728:0:14818:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:1.0:1773941617.922731:0:14818:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:1.0:1773941617.922736:0:14818:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941617.922740:0:14818:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:1.0:1773941617.922749:0:14818:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:1.0:1773941617.922753:0:14818:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:1.0:1773941617.922757:0:14818:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941617.922760:0:14818:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941617.922763:0:14818:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941617.922767:0:14818:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941617.922768:0:14818:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941617.922771:0:14818:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:1.0:1773941617.922795:0:14818:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941617.922797:0:14818:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941617.922799:0:14818:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941617.922804:0:14818:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:1.0:1773941617.922809:0:14818:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941617.922813:0:14818:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941617.922815:0:14818:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941617.922819:0:14818:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941617.922822:0:14818:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941618.491062:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1773941618.491074:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941618.491082:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941618.491086:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:0.0:1773941618.491088:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941618.491090:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:01000000:0.0:1773941618.491094:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:0.0:1773941618.491097:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:0.0:1773941618.491101:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:0.0:1773941618.491105:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1773941618.491107:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:0.0:1773941618.491120:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10836680. 00000020:00000040:0.0:1773941618.491127:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941618.491135:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941618.491140:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941618.491149:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941618.491160:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065c5a00. 00000100:00000001:0.0:1773941618.491171:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941618.491178:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000001:0.0:1773941618.491183:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000010:0.0:1773941618.491194:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d21ec1400. 00010000:00000001:0.0:1773941618.491202:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1773941618.491208:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d00031400. 00000020:00000001:0.0:1773941618.491214:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:0.0:1773941618.491218:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d00031400 with handle 0x8e86767622e2af6 to hash 00000020:00000001:0.0:1773941618.491233:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:0.0:1773941618.491236:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330090853376 : -113743618698240 : ffff988d00031400) 00010000:00000001:0.0:1773941618.491241:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330090853376 : -113743618698240 : ffff988d00031400) 00010000:00000001:0.0:1773941618.491248:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941618.491252:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941618.491258:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e2af6 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941618.491273:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e2af6 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:0.0:1773941618.491293:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e2af6 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941618.491318:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:0.0:1773941618.491321:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941618.491330:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb6840. 00000100:00000001:0.0:1773941618.491333:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330139633728 : -113743569917888 : ffff988d02eb6840) 00000100:00000001:0.0:1773941618.491337:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941618.491340:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367698560 : -113743341853056 : ffff988d10836680) 00000100:00000001:0.0:1773941618.491347:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941618.491354:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941618.491357:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941618.491359:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941618.491375:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10836680 x1860112291500160/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941618.491386:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941618.491389:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941618.491395:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1773941618.491398:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941618.491401:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941618.491405:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10836680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291500160:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:0.0:1773941618.491423:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941618.491432:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1773941618.491436:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941618.491440:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:0.0:1773941618.491446:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d02ce5000. 02000000:00000001:0.0:1773941618.491448:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941618.491460:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941618.491466:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0014baa8. 00000400:00000010:0.0:1773941618.491475:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d108336e8. 00000100:00000200:0.0:1773941618.491487:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291500160, portal 25 00000100:00000001:0.0:1773941618.491489:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941618.491491:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367698560 : -113743341853056 : ffff988d10836680) 00000100:00000040:0.0:1773941618.491499:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10836680 x1860112291500160/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941634 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941618.491509:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:0.0:1773941618.491513:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:0.0:1773941618.491516:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d10833000. 00000100:00000200:0.0:1773941618.491520:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291500160, offset 0 00000400:00000200:0.0:1773941618.491529:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941618.491541:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941618.491553:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941618.491562:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941618.491568:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941618.491574:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941618.491578:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941618.491582:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[184, 185] 00000400:00000200:0.0:1773941618.491587:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:0.0:1773941618.491592:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [296:296:256:4294967295] 192.168.204.114@tcp LPNI seq info [186:186:8:4294967295] 00000400:00000200:0.0:1773941618.491598:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941618.491611:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:0.0:1773941618.491619:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.114@tcp 00000800:00000010:0.0:1773941618.491631:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d113e9800. 00000800:00000200:0.0:1773941618.491641:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:0.0:1773941618.491653:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:0.0:1773941618.491660:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d113e9800 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1773941618.491705:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941618.491711:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941618.491716:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4295010087 00000100:00000001:0.0:1773941618.491728:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4295009515 00000100:00000001:0.0:1773941618.491734:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4295010087 00000100:00000001:0.0:1773941618.491739:0:14708:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4295009515 00000100:00000001:0.0:1773941618.491743:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1773941618.491747:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941618.491750:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941618.491753:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:0.0:1773941618.491757:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6840 going to sleep for 11 seconds 00000100:00000001:0.0:1773941618.491761:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941618.491766:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941618.491936:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941618.491945:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d113e9800. 00000400:00000200:1.0:1773941618.492032:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941618.492042:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941618.492050:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d10833000 00000400:00000010:1.0:1773941618.492073:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d10833000. 00000100:00000001:1.0:1773941618.492080:0:2394:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941618.492088:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10836680 x1860112291500160/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941634 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941618.492104:0:2394:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941618.492109:0:2394:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d10836680 x1860112291500160/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941634 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941618.492146:0:2394:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941618.492150:0:2394:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:1.0:1773941618.493868:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941618.493889:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941618.493895:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941618.493900:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941618.493909:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941618.493917:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941618.493927:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941618.493933:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941618.493940:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941618.493946:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324609080 00000400:00000200:1.0:1773941618.494009:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941618.494019:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0x12b9 [1] + 224 00000800:00000001:1.0:1773941618.494030:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941618.494044:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941618.494047:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941618.494052:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:1.0:1773941618.494059:0:2393:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941618.494066:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10836680 x1860112291500160/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941634 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941618.494081:0:2393:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d10836680 x1860112291500160/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941634 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941618.494107:0:2393:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1773941618.494119:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941618.494123:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941618.494167:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941618.494179:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941618.494182:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941618.494186:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0014baa8. 00000400:00000200:0.0:1773941618.494192:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d108336e8 00000400:00000010:0.0:1773941618.494195:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d108336e8. 00000100:00000001:0.0:1773941618.494199:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941618.494208:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10836680 x1860112291500160/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941634 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941618.494224:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10836680 x1860112291500160/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941634 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941618.494235:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941618.494237:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941618.494240:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941618.494244:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941618.494248:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941618.494263:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941618.494267:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941618.494276:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=119 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941618.494281:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941618.494285:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:0.0:1773941618.494291:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:0.0:1773941618.494313:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941618.494318:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:0.0:1773941618.494321:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:0.0:1773941618.494327:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:0.0:1773941618.494333:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:0.0:1773941618.494338:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:0.0:1773941618.494341:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:0.0:1773941618.494344:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:0.0:1773941618.494347:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:0.0:1773941618.494351:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d10836680 x1860112291500160/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941634 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941618.494360:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:0.0:1773941618.494362:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:0.0:1773941618.494364:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941618.494368:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:0.0:1773941618.494371:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:0.0:1773941618.494373:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941618.494375:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941618.494377:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941618.494378:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:0.0:1773941618.494380:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941618.494382:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941618.494385:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1773941618.494387:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1773941618.494391:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1773941618.494394:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 340 00000400:00000200:0.0:1773941618.494401:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941618.494413:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:0.0:1773941618.494419:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941618.494423:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941618.494427:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941618.494430:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:0.0:1773941618.494435:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:0.0:1773941618.494437:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:0.0:1773941618.494440:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941618.494443:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:0.0:1773941618.494446:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:0.0:1773941618.494449:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:0.0:1773941618.494452:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941618.494455:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941618.494457:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941618.494460:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:0.0:1773941618.494464:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00080000:0.0:1773941618.494468:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:0.0:1773941618.494470:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941618.494474:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1773941618.494477:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1773941618.494481:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941618.494484:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:0.0:1773941618.494486:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1773941618.494491:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10835880. 00000020:00000040:0.0:1773941618.494494:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 02000000:00000001:0.0:1773941618.494497:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1773941618.494500:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941618.494503:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941618.494506:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941618.494508:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941618.494516:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d02ce4800. 00000100:00000001:0.0:1773941618.494521:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941618.494529:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d10835880 x1860112291500288/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1773941618.494545:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941618.494549:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:0.0:1773941618.494555:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c1418601] to pc [ptlrpcd_rcv+-1] req@ffff988d10835880 x1860112291500288/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941618.494584:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941618.494588:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:0.0:1773941618.494590:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941618.494593:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941618.494596:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941618.494598:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1773941618.494601:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6840 going to sleep for 1 seconds 00000100:00000001:0.0:1773941618.494604:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941618.494607:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1773941618.494614:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10836680 x1860112291500160/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941634 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941618.494623:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:0.0:1773941618.494627:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941618.494631:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941618.494631:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941618.494636:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941618.494644:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1773941618.494645:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10836680 x1860112291500160/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941634 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941618.494653:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00100000:0.0:1773941618.494664:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10836680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291500160:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000040:3.0:1773941618.494675:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10835880 x1860112291500288/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941618.494675:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941618.494681:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941618.494687:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1773941618.494690:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941618.494700:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10836680 x1860112291500160/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941634 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941618.494716:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941618.494723:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941618.494729:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941618.494732:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941618.494737:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941618.494744:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:0.0:1773941618.494750:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb6840. 00000100:00100000:3.0:1773941618.494754:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10835880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291500288:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:0.0:1773941618.494760:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:3.0:1773941618.494762:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1773941618.494769:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 02000000:00000001:3.0:1773941618.494770:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941618.494775:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941618.494782:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 02000000:00000001:3.0:1773941618.494783:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941618.494794:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03522400. 00010000:00000001:0.0:1773941618.494795:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 02000000:00000001:3.0:1773941618.494801:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941618.494804:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000200:3.0:1773941618.494811:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000020:00000040:0.0:1773941618.494813:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00031400 refcount=4 00000400:00000010:3.0:1773941618.494817:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00000020:00000001:0.0:1773941618.494820:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090853376 : -113743618698240 : ffff988d00031400) 00010000:00000001:0.0:1773941618.494827:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090853376 : -113743618698240 : ffff988d00031400) 00000400:00000010:3.0:1773941618.494828:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00010000:00010000:0.0:1773941618.494836:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e2af6 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:3.0:1773941618.494837:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291500288, portal 25 00000100:00000001:3.0:1773941618.494844:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941618.494849:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367694976 : -113743341856640 : ffff988d10835880) 00010000:00000001:0.0:1773941618.494861:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000040:3.0:1773941618.494862:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10835880 x1860112291500288/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941628 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:0.0:1773941618.494871:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e2af6 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941618.494889:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941618.494898:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00010000:00000001:0.0:1773941618.494898:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00000400:00000010:3.0:1773941618.494908:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acc38. 00010000:00010000:0.0:1773941618.494909:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e2af6 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000200:3.0:1773941618.494913:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291500288, offset 0 00000400:00000200:3.0:1773941618.494928:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00010000:00000001:0.0:1773941618.494929:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941618.494934:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:3.0:1773941618.494939:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00010000:0.0:1773941618.494941:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e2af6 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941618.495021:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941618.495046:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00000001:0.0:1773941618.495048:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:3.0:1773941618.495055:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00010000:00000001:0.0:1773941618.495056:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941618.495064:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:0.0:1773941618.495065:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00000400:00000200:3.0:1773941618.495072:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00010000:0.0:1773941618.495074:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e2af6 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941618.495084:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[186, 185] 00010000:00010000:0.0:1773941618.495100:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d00031400 already unused, calling callback (ffffffffc12b4c20) 00000400:00000200:3.0:1773941618.495101:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:3.0:1773941618.495106:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [297:297:256:4294967295] 192.168.204.124@tcp LPNI seq info [187:187:8:4294967295] 00000400:00000200:3.0:1773941618.495114:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 10000000:00000001:0.0:1773941618.495123:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:0.0:1773941618.495130:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e2af6 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941618.495136:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:3.0:1773941618.495156:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.124@tcp 00010000:00000001:0.0:1773941618.495163:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000800:00000010:3.0:1773941618.495165:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74500. 00010000:00000001:0.0:1773941618.495171:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000800:00000200:3.0:1773941618.495180:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000020:00000001:0.0:1773941618.495181:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941618.495189:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00031400 refcount=5 00000800:00000200:3.0:1773941618.495195:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000020:00000001:0.0:1773941618.495201:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090853376 : -113743618698240 : ffff988d00031400) 00000800:00000200:3.0:1773941618.495203:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74500 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:0.0:1773941618.495217:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090853376 : -113743618698240 : ffff988d00031400) 00010000:00000001:0.0:1773941618.495228:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000100:00000001:3.0:1773941618.495231:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941618.495236:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e2af6 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941618.495241:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:3.0:1773941618.495249:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4295010091 00000100:00000001:3.0:1773941618.495260:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4295010087 10000000:00000001:0.0:1773941618.495269:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000001:3.0:1773941618.495271:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4295010091 10000000:00010000:0.0:1773941618.495281:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e2af6 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941618.495282:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4295010087 00000100:00000001:3.0:1773941618.495312:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941618.495324:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 10000000:01000000:0.0:1773941618.495326:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 10000000:00000040:0.0:1773941618.495334:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 10000000:00000001:0.0:1773941618.495342:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941618.495343:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941618.495351:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000100:00000001:3.0:1773941618.495353:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1773941618.495360:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00010000:00000001:0.0:1773941618.495367:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:0.0:1773941618.495370:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00000100:00000001:3.0:1773941618.495371:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00010000:00000001:0.0:1773941618.495375:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:3.0:1773941618.495377:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000800:00000200:1.0:1773941618.495379:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00000001:0.0:1773941618.495381:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941618.495388:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000010:1.0:1773941618.495390:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74500. 00000020:00000040:0.0:1773941618.495391:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d00031400 with handle 0x8e86767622e2af6 from hash 00000100:00000001:3.0:1773941618.495394:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00010000:00000001:0.0:1773941618.495402:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000100:00000001:3.0:1773941618.495406:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941618.495407:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000001:0.0:1773941618.495407:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941618.495412:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941618.495419:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000100:00000001:3.0:1773941618.495421:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941618.495426:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000400:00000200:1.0:1773941618.495429:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00010000:00000001:0.0:1773941618.495432:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000400:00000200:1.0:1773941618.495437:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acc38 00010000:00000001:0.0:1773941618.495439:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000400:00000010:1.0:1773941618.495452:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acc38. 00010000:00000001:0.0:1773941618.495457:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:1.0:1773941618.495461:0:2394:0:(events.c:34:request_out_callback()) Process entered 00010000:00000001:0.0:1773941618.495462:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941618.495468:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:1.0:1773941618.495473:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10835880 x1860112291500288/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941628 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:0.0:1773941618.495476:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e2af6 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1773941618.495512:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:1.0:1773941618.495513:0:2394:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00010000:00000001:0.0:1773941618.495515:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941618.495521:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00000100:00000040:1.0:1773941618.495523:0:2394:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10835880 x1860112291500288/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941628 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1773941618.495525:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1773941618.495532:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941618.495537:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000100:00000001:1.0:1773941618.495542:0:2394:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941618.495545:0:2394:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:0.0:1773941618.495550:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00010000:0.0:1773941618.495556:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e2af6 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000040:0.0:1773941618.495566:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d21ec1400 count: 0 00010000:00000001:0.0:1773941618.495589:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941618.495592:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941618.495596:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941618.495604:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10836680 x1860112291500160/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941634 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941618.495614:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1773941618.495617:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:0.0:1773941618.495621:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d02ce5000. 02000000:00000001:0.0:1773941618.495626:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1773941618.495628:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941618.495631:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000001:0.0:1773941618.495633:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 02000000:00000010:0.0:1773941618.495637:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065c5a00. 02000000:00000001:0.0:1773941618.495642:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1773941618.495645:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1773941618.495647:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10836680. 00000100:00000001:0.0:1773941618.495651:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1773941618.495653:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000001:0.0:1773941618.495655:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:0.0:1773941618.495664:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:0.0:1773941618.495668:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 10000000:00000010:0.0:1773941618.495673:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d10872940. 00000020:00000010:0.0:1773941618.495678:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d065c5a00. 00000020:00000010:0.0:1773941618.495685:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d065c5e00. 00000040:00000010:0.0:1773941618.495690:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d065c5400. 00000020:00000010:0.0:1773941618.495697:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d054a7000. 00000020:00000010:0.0:1773941618.495703:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d065c4a00. 80000000:00000010:0.0:1773941618.495709:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d06580b40. 00040000:00000010:0.0:1773941618.495716:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d065c4200. 20000000:00000010:0.0:1773941618.495726:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d065c5c00. 10000000:00000040:0.0:1773941618.495735:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:0.0:1773941618.495739:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:0.0:1773941618.495747:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d065c5e00. 00000040:00000010:0.0:1773941618.495754:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d065c5400. 00000020:00000010:0.0:1773941618.495760:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d054a7000. 00000020:00000010:0.0:1773941618.495766:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d065c4a00. 80000000:00000010:0.0:1773941618.495774:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d06580b40. 00040000:00000010:0.0:1773941618.495779:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d065c4200. 20000000:00000010:0.0:1773941618.495784:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d065c5c00. 00000020:00000010:0.0:1773941618.495790:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d065c5a00. 10000000:00000010:0.0:1773941618.495794:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d10872940. 10000000:01000000:0.0:1773941618.495800:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:0.0:1773941618.495805:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:0.0:1773941618.495810:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:0.0:1773941618.495814:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1773941618.495816:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1773941618.495819:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941618.495824:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:1.0:1773941618.497592:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941618.497607:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941618.497612:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941618.497618:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941618.497627:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941618.497654:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:1.0:1773941618.497666:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941618.497673:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941618.497680:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941618.497686:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324609100 00000400:00000200:1.0:1773941618.497695:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941618.497704:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0x12c9 [1] + 0 00000800:00000001:1.0:1773941618.497715:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941618.497727:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941618.497732:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941618.497740:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:1.0:1773941618.497747:0:2393:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941618.497759:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10835880 x1860112291500288/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941628 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941618.497781:0:2393:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d10835880 x1860112291500288/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941628 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941618.497818:0:2393:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1773941618.497845:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941618.497853:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941618.497860:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941618.497877:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941618.497883:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941618.497886:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941618.497889:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941618.497897:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941618.497900:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941618.497904:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941618.497912:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10835880 x1860112291500288/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941628 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941618.497926:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10835880 x1860112291500288/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941628 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941618.497936:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941618.497938:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941618.497940:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941618.497943:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941618.497945:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941618.498010:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941618.498015:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941618.498022:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=119 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941618.498027:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941618.498030:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941618.498033:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941618.498036:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941618.498040:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941618.498044:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941618.498051:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10835880 x1860112291500288/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941628 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941618.498062:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941618.498065:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941618.498067:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941618.498071:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941618.498077:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941618.498079:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941618.498081:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941618.498083:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941618.498096:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941618.498099:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941618.498103:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941618.498108:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941618 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941618.498113:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941618.498122:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941618.498125:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941618.498127:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941618.498130:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941618.498132:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00080000:3.0:1773941618.498138:0:2400:0:(import.c:1288:ptlrpc_connect_interpret()) reconnected to MGS@192.168.204.114@tcp after partition 00000100:00080000:3.0:1773941618.498141:0:2400:0:(import.c:1293:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: reconnected but import is invalid; marking evicted 00000100:00080000:3.0:1773941618.498145:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941618.498149:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941618.498153:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941618.498156:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941618.498582:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941618.498588:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941618.498596:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10835880 x1860112291500288/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941628 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941618.498607:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10835880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291500288:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:3.0:1773941618.498615:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941618.498619:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941618.498624:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10835880 x1860112291500288/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941628 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941618.498657:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000100:00000001:1.0:1773941618.498658:0:14819:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 02000000:00000001:3.0:1773941618.498662:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:3.0:1773941618.498671:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03522400. 02000000:00000001:3.0:1773941618.498678:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:1.0:1773941618.498680:0:14819:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000020:00000001:3.0:1773941618.498684:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000100:00100000:1.0:1773941618.498691:0:14819:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:1.0:1773941618.498698:0:14819:0:(obd_class.h:1215:obd_import_event()) Process entered 00000020:00000040:3.0:1773941618.498705:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 10000000:01000000:1.0:1773941618.498705:0:14819:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:1.0:1773941618.498716:0:14819:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000020:00000001:3.0:1773941618.498718:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000001:00000001:1.0:1773941618.498725:0:14819:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000010:3.0:1773941618.498727:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d02ce4800. 00000001:00000001:1.0:1773941618.498736:0:14819:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941618.498739:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000001:00000001:1.0:1773941618.498748:0:14819:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 02000000:00000001:3.0:1773941618.498749:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000001:00000001:1.0:1773941618.498754:0:14819:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000010:3.0:1773941618.498757:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10835880. 00000001:00000001:1.0:1773941618.498761:0:14819:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941618.498769:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 10000000:00000001:1.0:1773941618.498771:0:14819:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941618.498776:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941618.498781:0:14819:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941618.498783:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941618.498793:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:1.0:1773941618.498795:0:14819:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:3.0:1773941618.498796:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941618.498804:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1773941618.498808:0:14819:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00000001:3.0:1773941618.498809:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941618.498819:0:14819:0:(recover.c:147:ptlrpc_resend()) Process entered 00000100:00000001:1.0:1773941618.498827:0:14819:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1773941618.498833:0:14819:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 00000100:00000001:1.0:1773941618.498839:0:14819:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:1.0:1773941618.498842:0:14819:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 10000000:00000040:1.0:1773941618.498846:0:14819:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00080000:1.0:1773941618.498890:0:14819:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:1.0:1773941618.498895:0:14819:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941618.498900:0:14819:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000000:1.0:1773941618.498908:0:14819:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:1.0:1773941618.498916:0:14819:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1773941618.498921:0:14819:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:1.0:1773941618.498924:0:14819:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:1.0:1773941618.498929:0:14819:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941618.498932:0:14819:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000010:0.1:1773941618.501024:0:0:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d21ec1400. 00010000:00000010:0.1:1773941618.501037:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d00031400. 00000020:00000001:1.0:1773941618.683246:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1773941618.683258:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941618.683266:0:14706:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1773941619.707234:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1773941619.707240:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941619.707245:0:14705:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000040:0.0:1773941619.771142:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4295010091, at 4295010112 00000100:00000040:1.0:1773941619.771143:0:53:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4295010091, at 4295010112 00000100:00000010:0.0:1773941619.771166:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3d80. 00000100:00000010:1.0:1773941619.771188:0:53:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d054f3e40. 10000000:00000001:0.0:1773941619.963096:0:14708:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:0.0:1773941619.963110:0:14708:0:(mgc_request.c:108:config_log_get()) log lustre-sptlrpc (ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941619.963150:0:14708:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941619.963156:0:14694:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:00000001:0.0:1773941619.963179:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:01000000:2.0:1773941619.963180:0:14694:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-client-ffff988d111b1800 from 1 10000000:00000001:0.0:1773941619.963188:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:0.0:1773941619.963196:0:14708:0:(mgc_request.c:601:do_requeue()) Process entered 10000000:00000001:2.0:1773941619.963200:0:14694:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1773941619.963201:0:14708:0:(mgc_request.c:611:do_requeue()) updating log lustre-sptlrpc 10000000:00000001:0.0:1773941619.963209:0:14708:0:(mgc_request.c:1794:mgc_process_log()) Process entered 10000000:01000000:2.0:1773941619.963211:0:14694:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-client (res 0x65727473756c) 10000000:01000000:0.0:1773941619.963213:0:14708:0:(mgc_request.c:1813:mgc_process_log()) Process log lustre-sptlrpc-ffff988d2039cd28 from 1 10000000:00000001:0.0:1773941619.963242:0:14708:0:(mgc_request.c:990:mgc_enqueue()) Process entered 10000000:01000000:0.0:1773941619.963246:0:14708:0:(mgc_request.c:996:mgc_enqueue()) Enqueue for lustre-sptlrpc (res 0x65727473756c) 00000100:00000010:2.0:1773941619.963248:0:14694:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06506a00. 00000100:00000010:0.0:1773941619.963259:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10836680. 00000020:00000040:2.0:1773941619.963262:0:14694:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000020:00000040:0.0:1773941619.963268:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941619.963277:0:14694:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1773941619.963278:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941619.963287:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941619.963289:0:14694:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1773941619.963299:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941619.963303:0:14694:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941619.963314:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d065c5a00. 02000000:00000010:2.0:1773941619.963323:0:14694:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 512 at ffff988d03356200. 00000100:00000001:0.0:1773941619.963329:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941619.963339:0:14708:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00000100:00000001:2.0:1773941619.963341:0:14694:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941619.963345:0:14708:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00010000:00000001:2.0:1773941619.963353:0:14694:0:(ldlm_request.c:979:ldlm_cli_enqueue()) Process entered 00010000:00000010:0.0:1773941619.963361:0:14708:0:(ldlm_resource.c:1475:ldlm_resource_new()) slab-alloced 'res': 248 at ffff988d21ec1400. 00010000:00000001:0.0:1773941619.963372:0:14708:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00000010:0.0:1773941619.963377:0:14708:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d00031400. 00000020:00000001:0.0:1773941619.963389:0:14708:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00010000:00000001:2.0:1773941619.963391:0:14694:0:(ldlm_lock.c:1700:ldlm_lock_create()) Process entered 00000020:00000040:0.0:1773941619.963396:0:14708:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d00031400 with handle 0x8e86767622e2afd to hash 00000020:00000001:0.0:1773941619.963402:0:14708:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:0.0:1773941619.963406:0:14708:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330090853376 : -113743618698240 : ffff988d00031400) 00010000:00000040:2.0:1773941619.963408:0:14694:0:(ldlm_resource.c:1620:ldlm_resource_getref()) getref res: ffff988d21ec1400 count: 2 00010000:00000001:0.0:1773941619.963412:0:14708:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330090853376 : -113743618698240 : ffff988d00031400) 00010000:00000001:2.0:1773941619.963424:0:14694:0:(ldlm_resource.c:1559:ldlm_resource_get()) Process leaving via found (rc=0 : 0 : 0x0) 00010000:00000001:0.0:1773941619.963427:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:0.0:1773941619.963434:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941619.963438:0:14694:0:(ldlm_lock.c:433:ldlm_lock_new()) Process entered 00010000:00010000:0.0:1773941619.963444:0:14708:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e2afd lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000010:2.0:1773941619.963448:0:14694:0:(ldlm_lock.c:437:ldlm_lock_new()) slab-alloced 'lock': 512 at ffff988d089de400. 00000020:00000001:2.0:1773941619.963468:0:14694:0:(lustre_handles.c:45:class_handle_hash()) Process entered 00000020:00000040:2.0:1773941619.963476:0:14694:0:(lustre_handles.c:77:class_handle_hash()) added object ffff988d089de400 with handle 0x8e86767622e2b04 to hash 00010000:00010000:0.0:1773941619.963478:0:14708:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e2afd lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000020:00000001:2.0:1773941619.963489:0:14694:0:(lustre_handles.c:78:class_handle_hash()) Process leaving 00010000:00000001:2.0:1773941619.963497:0:14694:0:(ldlm_lock.c:488:ldlm_lock_new()) Process leaving (rc=18446630330235216896 : -113743474334720 : ffff988d089de400) 00010000:00000001:2.0:1773941619.963509:0:14694:0:(ldlm_lock.c:1741:ldlm_lock_create()) Process leaving (rc=18446630330235216896 : -113743474334720 : ffff988d089de400) 00010000:00000001:2.0:1773941619.963516:0:14694:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00010000:0.0:1773941619.963517:0:14708:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e2afd lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941619.963521:0:14694:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1773941619.963530:0:14694:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089de400/0x8e86767622e2b04 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x10000000000000 nid: local remote: 0x0 expref: -99 pid: 14694 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941619.963572:0:14708:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00010000:00010000:2.0:1773941619.963576:0:14694:0:(ldlm_request.c:1022:ldlm_cli_enqueue()) ### client-side enqueue START, flags 0x1000000000000 ns: MGC192.168.204.114@tcp lock: ffff988d089de400/0x8e86767622e2b04 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x0 nid: local remote: 0x0 expref: -99 pid: 14694 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941619.963581:0:14708:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000010:0.0:1773941619.963602:0:14708:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d02eb6840. 00010000:00010000:2.0:1773941619.963604:0:14694:0:(ldlm_request.c:1109:ldlm_cli_enqueue()) ### sending request ns: MGC192.168.204.114@tcp lock: ffff988d089de400/0x8e86767622e2b04 lrc: 3/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14694 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:0.0:1773941619.963616:0:14708:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330139633728 : -113743569917888 : ffff988d02eb6840) 00000100:00000001:2.0:1773941619.963620:0:14694:0:(client.c:3321:ptlrpc_queue_wait()) Process entered 00000100:00000001:2.0:1773941619.963626:0:14694:0:(client.c:1155:ptlrpc_prep_set()) Process entered 00000100:00000001:0.0:1773941619.963631:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000010:2.0:1773941619.963635:0:14694:0:(client.c:1157:ptlrpc_prep_set()) kmalloced '(set)': 144 at ffff988d09350540. 00000100:00000001:0.0:1773941619.963641:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367698560 : -113743341853056 : ffff988d10836680) 00000100:00000001:2.0:1773941619.963642:0:14694:0:(client.c:1172:ptlrpc_prep_set()) Process leaving (rc=18446630330245121344 : -113743464430272 : ffff988d09350540) 00000100:00000001:2.0:1773941619.963665:0:14694:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941619.963669:0:14694:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584960 : -113743512966656 : ffff988d06506a00) 00000100:00000001:0.0:1773941619.963671:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941619.963677:0:14694:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1773941619.963684:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941619.963686:0:14694:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941619.963709:0:14694:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:0.0:1773941619.963709:0:14708:0:(client.c:2663:ptlrpc_set_wait()) Process entered 00000100:00000001:2.0:1773941619.963722:0:14694:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1773941619.963724:0:14708:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:0.0:1773941619.963765:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10836680 x1860112291500416/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941619.963766:0:14694:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06506a00 x1860112291500544/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 0 ref 2 fl New:QU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941619.963900:0:14708:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1773941619.963904:0:14708:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941619.963910:0:14708:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1773941619.963912:0:14694:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1773941619.963915:0:14708:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941619.963952:0:14694:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941619.964026:0:14708:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941619.964029:0:14694:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1773941619.964036:0:14694:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1773941619.964042:0:14708:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10836680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291500416:192.168.204.114@tcp:101:ll_cfg_requeue.0 02000000:00000001:2.0:1773941619.964045:0:14694:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941619.964064:0:14708:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00100000:2.0:1773941619.964073:0:14694:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job mount.lustre:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14694:1860112291500544:192.168.204.114@tcp:101:mount.lustre.0 02000000:00000001:0.0:1773941619.964079:0:14708:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1773941619.964082:0:14694:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1773941619.964085:0:14708:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941619.964091:0:14708:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1773941619.964093:0:14694:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1773941619.964099:0:14694:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:0.0:1773941619.964101:0:14708:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d02ce5800. 02000000:00000001:2.0:1773941619.964105:0:14694:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1773941619.964108:0:14708:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941619.964116:0:14694:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d035a4800. 02000000:00000001:2.0:1773941619.964123:0:14694:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941619.964125:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:0.0:1773941619.964138:0:14708:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d0014baa8. 00000400:00000200:2.0:1773941619.964140:0:14694:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:2.0:1773941619.964156:0:14694:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d203795d8. 00000400:00000010:0.0:1773941619.964160:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d108336e8. 00000400:00000010:2.0:1773941619.964167:0:14694:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045162a8. 00000100:00000200:0.0:1773941619.964174:0:14708:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291500416, portal 25 00000100:00000200:2.0:1773941619.964179:0:14694:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291500544, portal 25 00000100:00000001:0.0:1773941619.964180:0:14708:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1773941619.964194:0:14694:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1773941619.964196:0:14708:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367698560 : -113743341853056 : ffff988d10836680) 00000100:00000001:2.0:1773941619.964209:0:14694:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584960 : -113743512966656 : ffff988d06506a00) 00000100:00000040:0.0:1773941619.964221:0:14708:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10836680 x1860112291500416/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941635 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:2.0:1773941619.964232:0:14694:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06506a00 x1860112291500544/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941635 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941619.964285:0:14694:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1773941619.964307:0:14708:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:2.0:1773941619.964309:0:14694:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000100:00000040:0.0:1773941619.964320:0:14708:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:2.0:1773941619.964326:0:14694:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d045167f8. 00000400:00000010:0.0:1773941619.964326:0:14708:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d108332a8. 00000100:00000200:0.0:1773941619.964344:0:14708:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291500416, offset 0 00000100:00000200:2.0:1773941619.964347:0:14694:0:(niobuf.c:94:ptl_send_buf()) Sending 328 bytes to portal 26, xid 1860112291500544, offset 0 00000400:00000200:0.0:1773941619.964358:0:14708:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941619.964370:0:14694:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:0.0:1773941619.964380:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941619.964402:0:14708:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:2.0:1773941619.964404:0:14694:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941619.964416:0:14708:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:2.0:1773941619.964426:0:14694:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:0.0:1773941619.964428:0:14708:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:0.0:1773941619.964446:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941619.964448:0:14694:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:0.0:1773941619.964465:0:14708:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:2.0:1773941619.964467:0:14694:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:2.0:1773941619.964484:0:14694:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941619.964486:0:14708:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[186, 187] 00000400:00000200:2.0:1773941619.964503:0:14694:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:0.0:1773941619.964509:0:14708:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00000400:00000200:2.0:1773941619.964527:0:14694:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[186, 187] 00000400:00000200:0.0:1773941619.964529:0:14708:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [298:298:256:4294967295] 192.168.204.114@tcp LPNI seq info [188:188:8:4294967295] 00000400:00000200:2.0:1773941619.964563:0:14694:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:0.0:1773941619.964571:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:2.0:1773941619.964581:0:14694:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [299:299:256:4294967295] 192.168.204.124@tcp LPNI seq info [189:189:8:4294967295] 00000400:00000200:2.0:1773941619.964610:0:14694:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:0.0:1773941619.964620:0:14708:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00000800:00000200:0.0:1773941619.964638:0:14708:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.114@tcp 00000400:00000200:2.0:1773941619.964655:0:14694:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000010:0.0:1773941619.964658:0:14708:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d113e9000. 00000800:00000200:2.0:1773941619.964669:0:14694:0:(socklnd_cb.c:1001:ksocknal_send()) sending 328 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:2.0:1773941619.964677:0:14694:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d0a0db900. 00000800:00000200:0.0:1773941619.964688:0:14708:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 00000800:00000200:2.0:1773941619.964698:0:14694:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:0.0:1773941619.964707:0:14708:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000800:00000200:2.0:1773941619.964717:0:14694:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:0.0:1773941619.964723:0:14708:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d113e9000 type 1, nob 424 niov 1 nkiov 1 00000800:00000200:2.0:1773941619.964726:0:14694:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d0a0db900 type 1, nob 424 niov 1 nkiov 1 00000100:00000001:0.0:1773941619.964756:0:14708:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941619.964767:0:14694:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:0.0:1773941619.964769:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941619.964780:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4295011560 00000400:00000200:2.0:1773941619.964781:0:14694:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:2.0:1773941619.964793:0:14694:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4295011560 00000100:00000010:0.0:1773941619.964799:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d06580b40. 00000100:00000001:0.0:1773941619.964816:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1773941619.964832:0:14708:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4295011560 00000100:00000001:2.0:1773941619.964833:0:14694:0:(niobuf.c:691:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, keep current deadline 4295011560 00000100:00000001:2.0:1773941619.964846:0:14694:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4295011560 00000100:00000010:0.0:1773941619.964852:0:14708:0:(niobuf.c:665:kick_cpu_latency()) kmalloced '(latency_qos->pm_qos_req)': 72 at ffff988d065805a0. 00000100:00000001:0.0:1773941619.964864:0:14708:0:(niobuf.c:677:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1773941619.964889:0:14708:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941619.964890:0:14694:0:(niobuf.c:691:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, keep current deadline 4295011560 00000100:00000001:2.0:1773941619.964904:0:14694:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941619.964912:0:14694:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941619.964913:0:14708:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941619.964917:0:14694:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1773941619.964943:0:14694:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00000001:0.0:1773941619.964944:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000800:00000200:1.0:1773941619.964947:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000100:00000001:0.0:1773941619.964950:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=11 : 11 : b) 00000100:00100000:2.0:1773941619.964953:0:14694:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d09350540 going to sleep for 11 seconds 00000100:00100000:0.0:1773941619.965013:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6840 going to sleep for 11 seconds 00000800:00000010:1.0:1773941619.965014:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d113e9000. 00000100:00000001:2.0:1773941619.965020:0:14694:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941619.965025:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1773941619.965030:0:14694:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941619.965031:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000100:00000001:0.0:1773941619.965033:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941619.965065:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000400:00000200:1.0:1773941619.965078:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d108332a8 00000400:00000010:1.0:1773941619.965084:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d108332a8. 00000100:00000001:1.0:1773941619.965092:0:2394:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941619.965109:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10836680 x1860112291500416/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941635 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941619.965126:0:2394:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941619.965132:0:2394:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d10836680 x1860112291500416/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941635 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941619.965142:0:2394:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941619.965146:0:2394:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000200:0.0:1773941619.965210:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:0.0:1773941619.965215:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d0a0db900. 00000400:00000200:0.0:1773941619.965223:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:0.0:1773941619.965247:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:0.0:1773941619.965254:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045167f8 00000400:00000010:0.0:1773941619.965258:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045167f8. 00000100:00000001:0.0:1773941619.965264:0:2393:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:0.0:1773941619.965271:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06506a00 x1860112291500544/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941635 ref 3 fl Rpc:r/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941619.965283:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:0.0:1773941619.965287:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 2 req@ffff988d06506a00 x1860112291500544/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941635 ref 3 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941619.965296:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941619.965299:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000001:1.0:1773941619.967052:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941619.967072:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941619.967076:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941619.967079:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941619.967088:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941619.967095:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941619.967105:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941619.967111:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941619.967119:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941619.967125:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324609180 00000400:00000200:1.0:1773941619.967131:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941619.967138:0:2394:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0x12d1 [1] + 224 00000800:00000001:1.0:1773941619.967149:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941619.967166:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941619.967172:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941619.967182:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:1.0:1773941619.967191:0:2394:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941619.967215:0:2394:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10836680 x1860112291500416/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941635 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941619.967238:0:2394:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=344 req@ffff988d10836680 x1860112291500416/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941635 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941619.967275:0:2394:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1773941619.967292:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941619.967299:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941619.967382:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:0.0:1773941619.967395:0:14708:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1773941619.967397:0:14708:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:0.0:1773941619.967403:0:14708:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0014baa8. 00000400:00000200:0.0:1773941619.967412:0:14708:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d108336e8 00000400:00000010:0.0:1773941619.967415:0:14708:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d108336e8. 00000100:00000001:0.0:1773941619.967421:0:14708:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:0.0:1773941619.967433:0:14708:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10836680 x1860112291500416/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941635 ref 2 fl Rpc:RQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:0.0:1773941619.967455:0:14708:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10836680 x1860112291500416/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941635 ref 2 fl Rpc:RQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941619.967470:0:14708:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1773941619.967489:0:14708:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941619.967495:0:14708:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:0.0:1773941619.967500:0:14708:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:0.0:1773941619.967505:0:14708:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:0.0:1773941619.967510:0:14708:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1773941619.967515:0:14708:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:0.0:1773941619.967526:0:14708:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=120 v=5 (5 0 0 0) 00000100:00001000:0.0:1773941619.967532:0:14708:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:0.0:1773941619.967538:0:14708:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:02020000:0.0:1773941619.967546:0:14708:0:(client.c:1490:ptlrpc_check_status()) MGC192.168.204.114@tcp: operation ldlm_enqueue to node 192.168.204.114@tcp failed: rc = -107 00000100:00000001:0.0:1773941619.967551:0:14708:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:0.0:1773941619.967557:0:14708:0:(recover.c:200:ptlrpc_request_handle_notconn()) Process entered 00000100:00080000:0.0:1773941619.967563:0:14708:0:(recover.c:203:ptlrpc_request_handle_notconn()) import MGC192.168.204.114@tcp of MGS@192.168.204.114@tcp abruptly disconnected: reconnecting 00000100:00080000:0.0:1773941619.967571:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to DISCONN 00000100:02020000:0.0:1773941619.967579:0:14708:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.204.114@tcp: Connection to MGS (at 192.168.204.114@tcp) was lost; in progress operations using this service will fail 00000100:00080000:0.0:1773941619.967587:0:14708:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.204.114@tcp for MGC192.168.204.114@tcp not replayable, auto-deactivating 00000100:00000001:0.0:1773941619.967592:0:14708:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:0.0:1773941619.967596:0:14708:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:0.0:1773941619.967600:0:14708:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00100000:0.0:1773941619.967609:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d10836680 x1860112291500416/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941635 ref 2 fl Rpc:RQU/200/0 rc 0/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1773941619.967630:0:14708:0:(client.c:3559:ptlrpc_abort_inflight()) @@@ inflight req@ffff988d06506a00 x1860112291500544/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941635 ref 2 fl Rpc:Qr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941619.967665:0:14708:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:0.0:1773941619.967669:0:14708:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:0.0:1773941619.967673:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941619.967678:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:0.0:1773941619.967683:0:14708:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:0.0:1773941619.967686:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941619.967690:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941619.967693:0:14708:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:0.0:1773941619.967695:0:14708:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:0.0:1773941619.967698:0:14708:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941619.967717:0:14708:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:0.0:1773941619.967723:0:14708:0:(import.c:700:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1773941619.967727:0:14708:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1773941619.967734:0:14708:0:(import.c:484:import_select_connection()) Process entered 00000100:00080000:0.0:1773941619.967742:0:14708:0:(import.c:509:import_select_connection()) MGC192.168.204.114@tcp: connect to NID 192.168.204.114@tcp last attempt 341 00000400:00000200:0.0:1773941619.967754:0:14708:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:0.0:1773941619.967761:0:14708:0:(peer.c:1521:LNetPeerDiscovered()) Peer NID 192.168.204.114@tcp is discovered: rc = 1 00000100:00000040:0.0:1773941619.967769:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000001:2.0:1773941619.967776:0:14694:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941619.967776:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941619.967795:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000400:00000001:2.0:1773941619.967796:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941619.967809:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1773941619.967810:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000001:0.0:1773941619.967822:0:14708:0:(genops.c:811:class_conn2export()) Process entered 00000400:00000010:2.0:1773941619.967824:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203795d8. 00000020:00000040:0.0:1773941619.967835:0:14708:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000400:00000200:2.0:1773941619.967839:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d045162a8 00000020:00000001:0.0:1773941619.967842:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000400:00000010:2.0:1773941619.967851:0:14694:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d045162a8. 00000020:00000040:0.0:1773941619.967853:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000100:00000001:2.0:1773941619.967864:0:14694:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:0.0:1773941619.967869:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000200:2.0:1773941619.967878:0:14694:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06506a00 x1860112291500544/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941635 ref 2 fl Rpc:EQr/200/ffffffff rc -5/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1773941619.967881:0:14708:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00000040:0.0:1773941619.967912:0:14708:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000100:00000200:2.0:1773941619.967943:0:14694:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06506a00 x1860112291500544/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941635 ref 2 fl Rpc:EQU/200/ffffffff rc -5/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941619.967946:0:14708:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000040:0.0:1773941619.968016:0:14708:0:(connection.c:161:ptlrpc_connection_addref()) conn=ffff988d08249140 refcount 3 to 192.168.204.114@tcp 00000100:00000001:0.0:1773941619.968024:0:14708:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446630330227265856 : -113743482285760 : ffff988d08249140) 00000020:00000040:0.0:1773941619.968042:0:14708:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:2.0:1773941619.968051:0:14694:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1773941619.968057:0:14694:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1773941619.968060:0:14708:0:(import.c:623:import_select_connection()) MGC192.168.204.114@tcp: import ffff988d20336000 using connection 192.168.204.114@tcp/192.168.204.114@tcp 00000100:00000001:0.0:1773941619.968070:0:14708:0:(import.c:627:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941619.968072:0:14694:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06506a00 x1860112291500544/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941635 ref 2 fl Rpc:EQU/200/ffffffff rc -5/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1773941619.968084:0:14708:0:(sec.c:1567:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1773941619.968096:0:14708:0:(sec.c:1607:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1773941619.968099:0:14694:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551611 : -5 : 0xfffffffffffffffb) 02000000:00000001:0.0:1773941619.968107:0:14708:0:(sec.c:1635:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941619.968111:0:14694:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941619.968115:0:14708:0:(obd_class.h:881:obd_reconnect()) Process entered 00000100:00000001:2.0:1773941619.968117:0:14694:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941619.968120:0:14708:0:(obd_class.h:888:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941619.968122:0:14694:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000010:0.0:1773941619.968133:0:14708:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d10836a00. 00000100:00000040:2.0:1773941619.968135:0:14694:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06506a00 x1860112291500544/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941635 ref 2 fl Interpret:EQU/200/ffffffff rc -5/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000020:00000040:0.0:1773941619.968147:0:14708:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=6 obd=MGC192.168.204.114@tcp 02000000:00000001:0.0:1773941619.968165:0:14708:0:(sec.c:2043:sptlrpc_sepol_get()) Process entered 00000100:00100000:2.0:1773941619.968172:0:14694:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job mount.lustre:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14694:1860112291500544:192.168.204.114@tcp:101:mount.lustre.0 02000000:00000001:0.0:1773941619.968185:0:14708:0:(sec.c:2053:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941619.968190:0:14694:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941619.968192:0:14708:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1773941619.968200:0:14708:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1773941619.968201:0:14694:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551611 : -5 : fffffffffffffffb) 02000000:00000001:0.0:1773941619.968209:0:14708:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941619.968210:0:14694:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:2.0:1773941619.968217:0:14694:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000010:0.0:1773941619.968223:0:14708:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 1024 at ffff988d02ce7000. 00000100:00000040:2.0:1773941619.968225:0:14694:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06506a00 x1860112291500544/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941635 ref 2 fl Complete:EQU/200/ffffffff rc -5/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941619.968236:0:14708:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941619.968257:0:14694:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1773941619.968263:0:14708:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff988d10836a00 x1860112291500672/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000010:2.0:1773941619.968269:0:14694:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d09350540. 00000100:00000001:0.0:1773941619.968290:0:14708:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941619.968291:0:14694:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1773941619.968299:0:14708:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941619.968301:0:14694:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551611 : -5 : fffffffffffffffb) 00000100:00000040:0.0:1773941619.968310:0:14708:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000f9e7d804] to pc [ptlrpcd_rcv+-1] req@ffff988d10836a00 x1860112291500672/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1773941619.968315:0:14694:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00010000:00000001:2.0:1773941619.968331:0:14694:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:2.0:1773941619.968340:0:14694:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:2.0:1773941619.968345:0:14694:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089de400 refcount=4 00000100:00000001:0.0:1773941619.968352:0:14708:0:(import.c:832:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941619.968353:0:14694:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235216896 : -113743474334720 : ffff988d089de400) 00000100:00000001:0.0:1773941619.968360:0:14708:0:(recover.c:219:ptlrpc_request_handle_notconn()) Process leaving 00000100:00000001:0.0:1773941619.968367:0:14708:0:(client.c:1698:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:2.0:1773941619.968369:0:14694:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235216896 : -113743474334720 : ffff988d089de400) 00000100:00000001:0.0:1773941619.968373:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1773941619.968385:0:14708:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941619.968389:0:14708:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=1 : 1 : 1) 00010000:00010000:2.0:1773941619.968393:0:14694:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d089de400/0x8e86767622e2b04 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14694 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00100000:0.0:1773941619.968393:0:14708:0:(client.c:2690:ptlrpc_set_wait()) set ffff988d02eb6840 going to sleep for 1 seconds 00000100:00000001:0.0:1773941619.968404:0:14708:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1773941619.968410:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000040:0.0:1773941619.968424:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10836680 x1860112291500416/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941635 ref 2 fl Rpc:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1773941619.968436:0:14708:0:(client.c:2126:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:0.0:1773941619.968443:0:14708:0:(client.c:2949:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941619.968448:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1773941619.968458:0:14708:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1773941619.968469:0:14708:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941619.968474:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941619.968488:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00010000:00000001:2.0:1773941619.968489:0:14694:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551611 : -5 : 0xfffffffffffffffb) 00000100:00000040:0.0:1773941619.968492:0:14708:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10836680 x1860112291500416/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941635 ref 2 fl Interpret:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00010000:2.0:1773941619.968509:0:14694:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d089de400/0x8e86767622e2b04 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14694 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000040:3.0:1773941619.968516:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d10836a00 x1860112291500672/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:2.0:1773941619.968571:0:14694:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:2.0:1773941619.968580:0:14694:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d089de400/0x8e86767622e2b04 lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14694 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000100:00000001:3.0:1773941619.968598:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00100000:0.0:1773941619.968603:0:14708:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10836680 pname:cluuid:pid:xid:nid:opc:job ll_cfg_requeue:c1c1ff21-5f68-4bf6-bdc4-258644b24465:14708:1860112291500416:192.168.204.114@tcp:101:ll_cfg_requeue.0 00000100:00000001:3.0:1773941619.968605:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941619.968615:0:14694:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 02000000:00000001:3.0:1773941619.968619:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00010000:00000001:2.0:1773941619.968621:0:14694:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 02000000:00000001:3.0:1773941619.968631:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00010000:00010000:2.0:1773941619.968634:0:14694:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d089de400/0x8e86767622e2b04 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14694 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941619.968637:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941619.968640:0:14708:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1773941619.968653:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d10836a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291500672:192.168.204.114@tcp:250:ll_cfg_requeue.0 00010000:00000001:2.0:1773941619.968654:0:14694:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00010000:00000001:2.0:1773941619.968663:0:14694:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941619.968673:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1773941619.968673:0:14708:0:(client.c:2754:ptlrpc_set_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00010000:00000001:2.0:1773941619.968676:0:14694:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 02000000:00000001:3.0:1773941619.968683:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 00010000:00010000:2.0:1773941619.968688:0:14694:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d089de400/0x8e86767622e2b04 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14694 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 02000000:00000001:3.0:1773941619.968690:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941619.968690:0:14708:0:(client.c:1223:ptlrpc_set_destroy()) Process entered 00000100:00000001:0.0:1773941619.968703:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1773941619.968705:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941619.968717:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 1024 at ffff988d03522400. 00010000:00010000:2.0:1773941619.968720:0:14694:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d089de400 already unused, calling callback (ffffffffc12b4c20) 00000100:00000040:0.0:1773941619.968726:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10836680 x1860112291500416/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941635 ref 2 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1773941619.968731:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941619.968733:0:14694:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 10000000:00010000:2.0:1773941619.968744:0:14694:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d089de400/0x8e86767622e2b04 lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14694 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941619.968769:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941619.968788:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00000400:00000010:3.0:1773941619.968809:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00010000:00000001:2.0:1773941619.968809:0:14694:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00000100:00000001:0.0:1773941619.968812:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000200:3.0:1773941619.968823:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 1024 bytes, xid 1860112291500672, portal 25 00010000:00000001:2.0:1773941619.968824:0:14694:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000100:00000010:0.0:1773941619.968827:0:14708:0:(client.c:3770:ptlrpc_reqset_free()) kfreed 'set': 144 at ffff988d02eb6840. 00000020:00000001:2.0:1773941619.968833:0:14694:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000100:00000001:3.0:1773941619.968836:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000020:00000040:2.0:1773941619.968840:0:14694:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d089de400 refcount=5 00000100:00000001:0.0:1773941619.968841:0:14708:0:(client.c:1260:ptlrpc_set_destroy()) Process leaving 00000100:00000001:0.0:1773941619.968853:0:14708:0:(client.c:3340:ptlrpc_queue_wait()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000020:00000001:2.0:1773941619.968854:0:14694:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330235216896 : -113743474334720 : ffff988d089de400) 00000100:00000001:3.0:1773941619.968856:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330367699456 : -113743341852160 : ffff988d10836a00) 00010000:00000001:2.0:1773941619.968872:0:14694:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330235216896 : -113743474334720 : ffff988d089de400) 00010000:00000001:0.0:1773941619.968872:0:14708:0:(ldlm_request.c:604:ldlm_cli_enqueue_fini()) Process entered 00000100:00000040:3.0:1773941619.968886:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d10836a00 x1860112291500672/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941629 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1773941619.968888:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:2.0:1773941619.968891:0:14694:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00000020:00000001:0.0:1773941619.968897:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:0.0:1773941619.968904:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00031400 refcount=4 00010000:00010000:2.0:1773941619.968941:0:14694:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d089de400/0x8e86767622e2b04 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14694 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000020:00000001:0.0:1773941619.968955:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090853376 : -113743618698240 : ffff988d00031400) 00000100:00000001:3.0:1773941619.968956:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00010000:00000001:0.0:1773941619.969018:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090853376 : -113743618698240 : ffff988d00031400) 00010000:00010000:0.0:1773941619.969026:0:14708:0:(ldlm_request.c:631:ldlm_cli_enqueue_fini()) ### client-side enqueue END (FAILED) ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e2afd lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x1000000000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:2.0:1773941619.969040:0:14694:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00010000:00000001:0.0:1773941619.969050:0:14708:0:(ldlm_request.c:634:ldlm_cli_enqueue_fini()) Process leaving via cleanup (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000040:3.0:1773941619.969052:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 10000000:00010000:2.0:1773941619.969058:0:14694:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d089de400/0x8e86767622e2b04 lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14694 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000010:3.0:1773941619.969073:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182ac330. 00010000:00010000:0.0:1773941619.969075:0:14708:0:(ldlm_request.c:548:failed_lock_cleanup()) ### setting FL_LOCAL_ONLY | LDLM_FL_FAILED | LDLM_FL_ATOMIC_CB | LDLM_FL_CBPENDING ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e2afd lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1773941619.969085:0:14694:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00000100:00000200:3.0:1773941619.969094:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 520 bytes to portal 26, xid 1860112291500672, offset 0 10000000:00000040:2.0:1773941619.969115:0:14694:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000400:00000200:3.0:1773941619.969130:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 10000000:00000001:2.0:1773941619.969132:0:14694:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941619.969132:0:14708:0:(ldlm_lock.c:854:ldlm_lock_decref_internal()) Process entered 00010000:00010000:0.0:1773941619.969150:0:14708:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(CR) ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e2afd lrc: 4/1,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941619.969163:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941619.969183:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00010000:00000001:0.0:1773941619.969184:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941619.969196:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000400:00000200:3.0:1773941619.969199:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00010000:00010000:0.0:1773941619.969204:0:14708:0:(ldlm_lock.c:887:ldlm_lock_decref_internal()) ### final decref done on CBPENDING lock ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e2afd lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x11206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941619.969211:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941619.969231:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00010000:00000001:0.0:1773941619.969243:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:3.0:1773941619.969246:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941619.969257:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[188, 189] 00010000:00000001:0.0:1773941619.969261:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941619.969272:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.114@tcp 00010000:00000001:0.0:1773941619.969274:0:14708:0:(ldlm_lockd.c:1905:ldlm_handle_bl_callback()) Process entered 00010000:00010000:2.0:1773941619.969277:0:14694:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00010000:00010000:0.0:1773941619.969288:0:14708:0:(ldlm_lockd.c:1907:ldlm_handle_bl_callback()) ### client blocking AST callback handler ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e2afd lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941619.969291:0:14694:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000400:00000200:3.0:1773941619.969302:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [300:300:256:4294967295] 192.168.204.114@tcp LPNI seq info [190:190:8:4294967295] 00010000:00000001:2.0:1773941619.969306:0:14694:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00010000:00000001:2.0:1773941619.969320:0:14694:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 00010000:00000001:2.0:1773941619.969328:0:14694:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000400:00000200:3.0:1773941619.969331:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00010000:00000001:2.0:1773941619.969335:0:14694:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:2.0:1773941619.969346:0:14694:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d089de400 with handle 0x8e86767622e2b04 from hash 00010000:00000001:2.0:1773941619.969357:0:14694:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 00000400:00000200:3.0:1773941619.969360:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.114@tcp(192.168.204.114@tcp:192.168.204.114@tcp) : PUT try# 0 00010000:00000001:2.0:1773941619.969363:0:14694:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941619.969369:0:14694:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941619.969374:0:14694:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 00000800:00000200:3.0:1773941619.969380:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 520 bytes in 1 frags to 12345-192.168.204.114@tcp 00010000:00000001:2.0:1773941619.969381:0:14694:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00010000:00010000:0.0:1773941619.969384:0:14708:0:(ldlm_lockd.c:1921:ldlm_handle_bl_callback()) Lock ffff988d00031400 already unused, calling callback (ffffffffc12b4c20) 00010000:00000001:2.0:1773941619.969389:0:14694:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 10000000:00000001:0.0:1773941619.969394:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00010000:00000001:2.0:1773941619.969398:0:14694:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000800:00000010:3.0:1773941619.969399:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74f00. 10000000:00010000:0.0:1773941619.969401:0:14708:0:(mgc_request.c:922:mgc_blocking_ast()) ### MGC blocking CB ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e2afd lrc: 4/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x1206400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:2.0:1773941619.969408:0:14694:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941619.969418:0:14694:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:3.0:1773941619.969433:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d21c930c0] -> 12345-192.168.204.114@tcp (5) 10000000:00000001:2.0:1773941619.969435:0:14694:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941619.969446:0:14708:0:(ldlm_request.c:1626:ldlm_cli_cancel()) Process entered 00010000:00010000:2.0:1773941619.969449:0:14694:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d089de400/0x8e86767622e2b04 lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14694 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00000001:0.0:1773941619.969451:0:14708:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00000020:00000001:0.0:1773941619.969460:0:14708:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000800:00000200:3.0:1773941619.969471:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.114@tcp ip 192.168.204.114:988 00000020:00000040:0.0:1773941619.969472:0:14708:0:(lustre_handles.c:131:class_handle2object()) GET ldlm ffff988d00031400 refcount=5 00010000:00000001:2.0:1773941619.969480:0:14694:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000020:00000001:0.0:1773941619.969483:0:14708:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330090853376 : -113743618698240 : ffff988d00031400) 00010000:00000001:2.0:1773941619.969488:0:14694:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00000800:00000200:3.0:1773941619.969489:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74f00 type 1, nob 616 niov 1 nkiov 1 00010000:00000001:0.0:1773941619.969494:0:14708:0:(ldlm_lock.c:634:__ldlm_handle2lock()) Process leaving (rc=18446630330090853376 : -113743618698240 : ffff988d00031400) 00010000:00000001:2.0:1773941619.969496:0:14694:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:2.0:1773941619.969503:0:14694:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1773941619.969516:0:14708:0:(ldlm_request.c:1280:ldlm_cli_cancel_local()) Process entered 00010000:00000001:2.0:1773941619.969517:0:14694:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:2.0:1773941619.969526:0:14694:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00010000:0.0:1773941619.969528:0:14708:0:(ldlm_request.c:1285:ldlm_cli_cancel_local()) ### client-side cancel ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e2afd lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x120e400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:00000001:0.0:1773941619.969540:0:14708:0:(mgc_request.c:917:mgc_blocking_ast()) Process entered 00000100:00000001:3.0:1773941619.969542:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941619.969543:0:14694:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 10000000:00010000:0.0:1773941619.969547:0:14708:0:(mgc_request.c:928:mgc_blocking_ast()) ### MGC cancel CB ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e2afd lrc: 5/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 3 type: PLN flags: 0x120f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00010000:00010000:2.0:1773941619.969552:0:14694:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d089de400/0x8e86767622e2b04 lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 4 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14694 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000400:00000200:3.0:1773941619.969567:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:3.0:1773941619.969581:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4295011565 00000100:00000001:3.0:1773941619.969605:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4295011560 00000100:00000001:3.0:1773941619.969626:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4295011565 10000000:01000000:0.0:1773941619.969628:0:14708:0:(mgc_request.c:932:mgc_blocking_ast()) Lock res [0x65727473756c:0x0:0x0].0x0 (lustre) 00010000:00000001:2.0:1773941619.969629:0:14694:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:2.0:1773941619.969635:0:14694:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551611 : -5 : fffffffffffffffb) 00000100:00000001:3.0:1773941619.969636:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4295011560 10000000:00000040:0.0:1773941619.969648:0:14708:0:(mgc_request.c:935:mgc_blocking_ast()) missing data, won't requeue 00000100:00000001:2.0:1773941619.969649:0:14694:0:(client.c:2842:__ptlrpc_req_put()) Process entered 10000000:00000001:0.0:1773941619.969663:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941619.969665:0:14694:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06506a00 x1860112291500544/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/344 e 0 to 0 dl 1773941635 ref 1 fl Complete:EQU/200/ffffffff rc -5/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941619.969667:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000200:1.0:1773941619.969680:0:2393:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00010000:00010000:0.0:1773941619.969683:0:14708:0:(ldlm_request.c:1301:ldlm_cli_cancel_local()) not sending request (at caller's instruction) 00000100:00000001:3.0:1773941619.969684:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941619.969694:0:14708:0:(ldlm_lock.c:2502:ldlm_lock_cancel()) Process entered 00000800:00000010:1.0:1773941619.969706:0:2393:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74f00. 00010000:00000001:0.0:1773941619.969708:0:14708:0:(ldlm_lock.c:410:ldlm_lock_destroy_nolock()) Process entered 00000100:00000001:2.0:1773941619.969712:0:14694:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000400:00000200:1.0:1773941619.969720:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00010000:00000001:0.0:1773941619.969721:0:14708:0:(ldlm_lock.c:354:ldlm_lock_destroy_internal()) Process entered 02000000:00000001:2.0:1773941619.969722:0:14694:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000010:2.0:1773941619.969736:0:14694:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d035a4800. 00000400:00000200:1.0:1773941619.969738:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00010000:00000001:0.0:1773941619.969739:0:14708:0:(ldlm_lock.c:261:ldlm_lock_remove_from_lru_check()) Process entered 00000100:00000001:3.0:1773941619.969746:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1773941619.969752:0:14694:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00010000:00000001:0.0:1773941619.969755:0:14708:0:(ldlm_lock.c:280:ldlm_lock_remove_from_lru_check()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:1.0:1773941619.969760:0:2393:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182ac330 00000020:00000001:2.0:1773941619.969761:0:14694:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:3.0:1773941619.969765:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000040:2.0:1773941619.969774:0:14694:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000020:00000040:0.0:1773941619.969774:0:14708:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d00031400 with handle 0x8e86767622e2afd from hash 00000400:00000010:1.0:1773941619.969775:0:2393:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182ac330. 00000100:00000001:3.0:1773941619.969787:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:2.0:1773941619.969795:0:14694:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:1.0:1773941619.969796:0:2393:0:(events.c:34:request_out_callback()) Process entered 00010000:00000001:0.0:1773941619.969807:0:14708:0:(ldlm_lock.c:384:ldlm_lock_destroy_internal()) Process leaving 02000000:00000010:2.0:1773941619.969808:0:14694:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d03356200. 00000100:00000200:1.0:1773941619.969812:0:2393:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d10836a00 x1860112291500672/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941629 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1773941619.969819:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941619.969831:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 02000000:00000001:2.0:1773941619.969832:0:14694:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1773941619.969838:0:2393:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00010000:00000001:0.0:1773941619.969843:0:14708:0:(ldlm_lock.c:416:ldlm_lock_destroy_nolock()) Process leaving 02000000:00000001:2.0:1773941619.969845:0:14694:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000040:1.0:1773941619.969854:0:2393:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d10836a00 x1860112291500672/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941629 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00010000:00000001:0.0:1773941619.969857:0:14708:0:(ldlm_lock.c:2538:ldlm_lock_cancel()) Process leaving 00000100:00000010:2.0:1773941619.969858:0:14694:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06506a00. 00010000:00000001:0.0:1773941619.969870:0:14708:0:(ldlm_request.c:1316:ldlm_cli_cancel_local()) Process leaving (rc=137438953472 : 137438953472 : 2000000000) 00000100:00000001:2.0:1773941619.969880:0:14694:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1773941619.969886:0:2393:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1773941619.969887:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00000100:00000001:2.0:1773941619.969889:0:14694:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1773941619.969891:0:2393:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:0.0:1773941619.969897:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 10000000:00000001:2.0:1773941619.969902:0:14694:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551611 : -5 : fffffffffffffffb) 00010000:00000001:0.0:1773941619.969903:0:14708:0:(ldlm_request.c:1665:ldlm_cli_cancel()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941619.969912:0:14708:0:(mgc_request.c:967:mgc_blocking_ast()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1773941619.969944:0:14708:0:(ldlm_lockd.c:1933:ldlm_handle_bl_callback()) ### client blocking callback handler END ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e2afd lrc: 3/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 10000000:01000000:2.0:1773941619.969948:0:14694:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -5 00010000:00000001:0.0:1773941619.970040:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941619.970045:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 00010000:00000001:0.0:1773941619.970050:0:14708:0:(ldlm_lockd.c:1935:ldlm_handle_bl_callback()) Process leaving 00010000:00000001:0.0:1773941619.970053:0:14708:0:(ldlm_lock.c:923:ldlm_lock_decref_internal()) Process leaving 00010000:00000001:0.0:1773941619.970057:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 00010000:00000001:0.0:1773941619.970061:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 10000000:00000001:2.0:1773941619.970066:0:14694:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00010000:00000001:0.0:1773941619.970068:0:14708:0:(ldlm_lock.c:185:ldlm_lock_put()) Process entered 10000000:00000010:2.0:1773941619.970078:0:14694:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d054378c0. 00010000:00010000:0.0:1773941619.970080:0:14708:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: MGC192.168.204.114@tcp lock: ffff988d00031400/0x8e86767622e2afd lrc: 0/0,0 mode: --/CR res: [0x65727473756c:0x0:0x0].0x0 rrc: 2 type: PLN flags: 0x5a0f400000000 nid: local remote: 0x0 expref: -99 pid: 14708 timeout: 0 lvb_type: 0 lru_score: 0 lru_type: 0 00000020:00000010:2.0:1773941619.970087:0:14694:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d03356200. 00000800:00000001:1.0:1773941619.970098:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000020:00000010:2.0:1773941619.970101:0:14694:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d01342200. 00010000:00000040:0.0:1773941619.970111:0:14708:0:(ldlm_resource.c:1668:ldlm_resource_putref()) putref res: ffff988d21ec1400 count: 0 00000040:00000010:2.0:1773941619.970112:0:14694:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d01343c00. 00000020:00000010:2.0:1773941619.970122:0:14694:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941619.970131:0:14694:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d0381e000. 80000000:00000010:2.0:1773941619.970137:0:14694:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d08bbb8a0. 00040000:00000010:2.0:1773941619.970146:0:14694:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d0381f200. 00010000:00000001:0.0:1773941619.970151:0:14708:0:(ldlm_lock.c:224:ldlm_lock_put()) Process leaving 20000000:00000010:2.0:1773941619.970153:0:14694:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d0381e800. 00000800:00000001:1.0:1773941619.970159:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00010000:00000001:0.0:1773941619.970162:0:14708:0:(ldlm_request.c:1133:ldlm_cli_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00000040:2.0:1773941619.970165:0:14694:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 00000100:00000001:3.0:1773941619.970166:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000800:00000001:1.0:1773941619.970170:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000100:00000001:0.0:1773941619.970177:0:14708:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1773941619.970183:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 10000000:00000001:2.0:1773941619.970185:0:14694:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551611 : -5 : 0xfffffffffffffffb) 00000800:00000001:1.0:1773941619.970190:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941619.970194:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000040:0.0:1773941619.970198:0:14708:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10836680 x1860112291500416/t0(0) o101->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 328/224 e 0 to 0 dl 1773941635 ref 1 fl Complete:ESQU/200/0 rc -107/-107 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000020:00000010:2.0:1773941619.970205:0:14694:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d01342200. 00000100:00000001:3.0:1773941619.970210:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000040:00000010:2.0:1773941619.970228:0:14694:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d01343c00. 00000400:00000200:1.0:1773941619.970230:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000100:00000001:3.0:1773941619.970232:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941619.970258:0:14708:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000020:00000010:2.0:1773941619.970261:0:14694:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d09305000. 02000000:00000001:0.0:1773941619.970267:0:14708:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000400:00000200:1.0:1773941619.970273:0:2394:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 02000000:00000010:0.0:1773941619.970280:0:14708:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d02ce5800. 00000020:00000010:2.0:1773941619.970285:0:14694:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d0381e000. 02000000:00000001:0.0:1773941619.970288:0:14708:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000400:00000200:1.0:1773941619.970294:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000020:00000001:0.0:1773941619.970297:0:14708:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941619.970305:0:14708:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 80000000:00000010:2.0:1773941619.970306:0:14694:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d08bbb8a0. 00000400:00000200:1.0:1773941619.970307:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000020:00000001:0.0:1773941619.970311:0:14708:0:(genops.c:1249:class_import_put()) Process leaving 00040000:00000010:2.0:1773941619.970328:0:14694:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d0381f200. 00000400:00000200:1.0:1773941619.970329:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 02000000:00000010:0.0:1773941619.970330:0:14708:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 512 at ffff988d065c5a00. 00000400:00000200:1.0:1773941619.970351:0:2394:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 344 into portal 25 MB=0x69bc324609200 02000000:00000001:0.0:1773941619.970353:0:14708:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 20000000:00000010:2.0:1773941619.970355:0:14694:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d0381e800. 02000000:00000001:0.0:1773941619.970361:0:14708:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:0.0:1773941619.970367:0:14708:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10836680. 00000400:00000200:1.0:1773941619.970368:0:2394:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:0.0:1773941619.970376:0:14708:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000020:00000010:2.0:1773941619.970378:0:14694:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d03356200. 00000100:00000001:0.0:1773941619.970380:0:14708:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 10000000:00000010:2.0:1773941619.970413:0:14694:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d054378c0. 10000000:00000001:0.0:1773941619.970417:0:14708:0:(mgc_request.c:1023:mgc_enqueue()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:01000000:2.0:1773941619.970426:0:14694:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-client' failed (-5). 00000400:00000100:1.0:1773941619.970428:0:2394:0:(lib-move.c:4515:lnet_parse_put()) Dropping PUT from 12345-192.168.204.114@tcp portal 25 match 1860112291500544 offset 224 length 344: 4 10000000:01000000:0.0:1773941619.970428:0:14708:0:(mgc_request.c:1828:mgc_process_log()) Can't get cfg lock: -107 10000000:00000001:0.0:1773941619.970444:0:14708:0:(mgc_request.c:1680:mgc_process_cfg_log()) Process entered 00000400:00000200:1.0:1773941619.970445:0:2394:0:(lib-msg.c:1043:lnet_is_health_check()) Msg ffff988d181dad60 is in inconsistent state, don't perform health checking (-2, 0) 10000000:00000001:2.0:1773941619.970448:0:14694:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551611 : -5 : fffffffffffffffb) 10000000:00000010:0.0:1773941619.970451:0:14708:0:(mgc_request.c:1694:mgc_process_cfg_log()) kmalloced '(env)': 56 at ffff988d10872940. 00000400:00000200:1.0:1773941619.970452:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 0, status = -2, hstatus = 0 10000000:00000001:2.0:1773941619.970461:0:14694:0:(mgc_request.c:2026:mgc_process_config()) Process leaving (rc=18446744073709551611 : -5 : fffffffffffffffb) 00000020:00000010:0.0:1773941619.970485:0:14708:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d065c5a00. 00000020:00000010:2.0:1773941619.970487:0:14694:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d03356400. 00000800:00000001:1.0:1773941619.970489:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941619.970508:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000040:00000010:2.0:1773941619.970511:0:14694:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d03356600. 00000020:00000010:0.0:1773941619.970514:0:14708:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d065c5c00. 00000800:00000001:1.0:1773941619.970522:0:2394:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000020:00000010:2.0:1773941619.970532:0:14694:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d054a7800. 00000040:00000010:0.0:1773941619.970534:0:14708:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d065c4200. 00000800:00000001:1.0:1773941619.970535:0:2394:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000020:00000010:2.0:1773941619.970557:0:14694:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d03357e00. 00000020:00000010:0.0:1773941619.970559:0:14708:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d054a7000. 00040000:00000010:2.0:1773941619.970575:0:14694:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d0a20be00. 00000020:00000010:0.0:1773941619.970581:0:14708:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d065c4a00. 00040000:00000010:2.0:1773941619.970582:0:14694:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d03357800. 00000020:00000010:2.0:1773941619.970596:0:14694:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d03356000. 80000000:00000010:0.0:1773941619.970602:0:14708:0:(fld_handler.c:35:fld_key_init()) kmalloced '(value)': 72 at ffff988d065806c0. 00000020:00000001:2.0:1773941619.970604:0:14694:0:(obd_class.h:696:obd_process_config()) Process leaving (rc=18446744073709551611 : -5 : fffffffffffffffb) 00000020:00000010:2.0:1773941619.970618:0:14694:0:(obd_mount.c:82:lustre_process_log()) kfreed 'lcfg': 184 at ffff988d02eb6300. 00000020:00000010:2.0:1773941619.970625:0:14694:0:(obd_mount.c:84:lustre_process_log()) kfreed 'bufs': 104 at ffff988d181fe500. 00040000:00000010:0.0:1773941619.970625:0:14708:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d065c5400. 00000020:02020000:2.0:1773941619.970651:0:14694:0:(obd_mount.c:91:lustre_process_log()) MGC192.168.204.114@tcp: Confguration from log lustre-client failed from MGS -5. Communication error between node & MGS, a bad configuration, or other errors. See syslog for more info 20000000:00000010:0.0:1773941619.970652:0:14708:0:(mgs_handler.c:1745:mgs_key_init()) kmalloced '(value)': 392 at ffff988d065c5e00. 10000000:00000040:0.0:1773941619.970667:0:14708:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 10000000:00000001:0.0:1773941619.970674:0:14708:0:(mgc_request.c:1726:mgc_process_cfg_log()) Process leaving via out_pop (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000020:00000010:0.0:1773941619.970684:0:14708:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d065c5c00. 00000040:00000010:0.0:1773941619.970690:0:14708:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d065c4200. 00000020:00000010:0.0:1773941619.970695:0:14708:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d054a7000. 00000020:00000010:0.0:1773941619.970700:0:14708:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d065c4a00. 80000000:00000010:0.0:1773941619.970704:0:14708:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d065806c0. 00040000:00000010:0.0:1773941619.970708:0:14708:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d065c5400. 20000000:00000010:0.0:1773941619.970711:0:14708:0:(mgs_handler.c:1745:mgs_key_fini()) kfreed 'info': 392 at ffff988d065c5e00. 00000020:00000010:0.0:1773941619.970715:0:14708:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d065c5a00. 10000000:00000010:0.0:1773941619.970718:0:14708:0:(mgc_request.c:1742:mgc_process_cfg_log()) kfreed 'env': 56 at ffff988d10872940. 10000000:01000000:0.0:1773941619.970722:0:14708:0:(mgc_request.c:1890:mgc_process_log()) MGC192.168.204.114@tcp: configuration from log 'lustre-sptlrpc' failed (-107). 10000000:00000001:0.0:1773941619.970726:0:14708:0:(mgc_request.c:1916:mgc_process_log()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 10000000:00020000:0.0:1773941619.970729:0:14708:0:(mgc_request.c:614:do_requeue()) failed processing log: -107 10000000:00000001:0.0:1773941619.970732:0:14708:0:(mgc_request.c:621:do_requeue()) Process leaving 10000000:00000001:0.0:1773941619.970734:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:0.0:1773941619.970745:0:14708:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 2 10000000:00000001:0.0:1773941619.970748:0:14708:0:(mgc_request.c:147:config_log_put()) Process leaving 00000800:00000001:1.0:1773941619.971542:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941619.971553:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941619.971556:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941619.971560:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941619.971565:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941619.971571:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.114@tcp : PUT - for me 00000400:00000200:1.0:1773941619.971578:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941619.971582:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941619.971586:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941619.971590:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 416 into portal 25 MB=0x69bc324609280 00000400:00000200:1.0:1773941619.971595:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941619.971601:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 416/416 into md 0x12e9 [1] + 0 00000800:00000001:1.0:1773941619.971608:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941619.971616:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941619.971619:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941619.971623:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.114@tcp: PUT: OK 00000100:00000001:1.0:1773941619.971628:0:2393:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941619.971635:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d10836a00 x1860112291500672/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941629 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941619.971648:0:2393:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=416 offset=0 replen=544 req@ffff988d10836a00 x1860112291500672/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941629 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941619.971675:0:2393:0:(events.c:163:reply_in_callback()) Process leaving 00000800:00000001:1.0:1773941619.971681:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941619.971685:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941619.971754:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941619.971762:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000400:00000001:3.0:1773941619.971770:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941619.971773:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941619.971778:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941619.971788:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941619.971792:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941619.971799:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941619.971812:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d10836a00 x1860112291500672/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941629 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941619.971834:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d10836a00 x1860112291500672/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/544 e 0 to 0 dl 1773941629 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941619.971849:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941619.971853:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941619.971858:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941619.971863:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941619.971867:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941619.971871:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941619.971876:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941619.971885:0:2400:0:(import.c:2136:obd_at_measure()) add 5 to ffff988d20336458 time=120 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941619.971893:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941619.971898:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941619.971901:0:2400:0:(client.c:1518:ptlrpc_check_status()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941619.971906:0:2400:0:(ldlm_request.c:1516:ldlm_cli_update_pool()) Process entered 00010000:00000001:3.0:1773941619.971909:0:2400:0:(ldlm_request.c:1520:ldlm_cli_update_pool()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941619.971915:0:2400:0:(client.c:1781:after_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941619.972037:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d10836a00 x1860112291500672/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941629 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941619.972054:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941619.972059:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941619.972063:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941619.972067:0:2400:0:(import.c:1053:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1773941619.972076:0:2400:0:(import.c:1123:ptlrpc_connect_interpret()) MGC192.168.204.114@tcp: connect to target with instance 0 00000020:00000001:3.0:1773941619.972081:0:2400:0:(genops.c:811:class_conn2export()) Process entered 00000020:00000040:3.0:1773941619.972085:0:2400:0:(genops.c:823:class_conn2export()) looking for export cookie 0x8e86767622e28d4 00000020:00000001:3.0:1773941619.972089:0:2400:0:(lustre_handles.c:113:class_handle2object()) Process entered 00000020:00000040:3.0:1773941619.972094:0:2400:0:(lustre_handles.c:131:class_handle2object()) GET export ffff988d111b0800 refcount=3 00000020:00000001:3.0:1773941619.972099:0:2400:0:(lustre_handles.c:138:class_handle2object()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000020:00000001:3.0:1773941619.972105:0:2400:0:(genops.c:825:class_conn2export()) Process leaving (rc=18446630330377635840 : -113743331915776 : ffff988d111b0800) 00000100:00001000:3.0:1773941619.972112:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=1773941619 v=0 (0 0 0 0) 00000100:00001000:3.0:1773941619.972118:0:2400:0:(import.c:2202:obd_at_measure()) AT ffff988d20336410 change: old=0 new=5 delta=5 (val=1) hist 1 0 0 0 00000020:00000040:3.0:1773941619.972128:0:2400:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 2 00000100:00000001:3.0:1773941619.972133:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941619.972137:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808005 10000000:00000001:3.0:1773941619.972141:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941619.972144:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:02000400:3.0:1773941619.972151:0:2400:0:(import.c:1275:ptlrpc_connect_interpret()) Evicted from MGS (at 192.168.204.114@tcp) after server handle changed from 0x349f38f79dc46411 to 0xd03f0241a6d94a04 00000020:00000001:2.0:1773941619.980478:0:14694:0:(obd_mount.c:93:lustre_process_log()) Process leaving (rc=18446744073709551611 : -5 : fffffffffffffffb) 00000080:00000001:2.0:1773941619.980487:0:14694:0:(llite_lib.c:1489:ll_fill_super()) Process leaving via out_debugfs (rc=18446744073709551611 : -5 : 0xfffffffffffffffb) 00000020:00000010:2.0:1773941619.980821:0:14694:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 576 at ffff988d02ce7c00. 00000020:00000010:2.0:1773941619.980827:0:14694:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff988d02ce4c00. 00000020:00000010:2.0:1773941619.980831:0:14694:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 1440 at ffff988d054a6000. 00000020:00000010:2.0:1773941619.980838:0:14694:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff988d02ce6400. 00000080:00000010:2.0:1773941619.980842:0:14694:0:(llite_lib.c:1531:ll_fill_super()) kfreed 'cfg': 88 at ffff988d065809c0. 00000080:00000001:2.0:1773941619.980858:0:14694:0:(llite_lib.c:1565:ll_put_super()) Process entered 00000080:00200000:2.0:1773941619.980861:0:14694:0:(llite_lib.c:1572:ll_put_super()) VFS Op: cfg_instance lustre-client-ffff988d111b1800 (sb ffff988d111b1800) 00000020:00000001:2.0:1773941619.980866:0:14694:0:(obd_mount.c:116:lustre_end_log()) Process entered 00000020:00000010:2.0:1773941619.980871:0:14694:0:(obd_mount.c:126:lustre_end_log()) kmalloced '(lcfg)': 176 at ffff988d09350540. 00000020:00000001:2.0:1773941619.980875:0:14694:0:(obd_class.h:681:obd_process_config()) Process entered 00000020:00000010:2.0:1773941619.980879:0:14694:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d0381e800. 00000020:00000010:2.0:1773941619.980884:0:14694:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d0381f200. 00000040:00000010:2.0:1773941619.980888:0:14694:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d0381e000. 00000020:00000010:2.0:1773941619.980892:0:14694:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941619.980898:0:14694:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d03356000. 00040000:00000010:2.0:1773941619.980903:0:14694:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d0a0db000. 00040000:00000010:2.0:1773941619.980908:0:14694:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d03357800. 10000000:00000001:2.0:1773941619.980913:0:14694:0:(mgc_request.c:1932:mgc_process_config()) Process entered 10000000:00000001:2.0:1773941619.980915:0:14694:0:(mgc_request_server.c:668:mgc_process_config_server()) Process entered 10000000:00000001:2.0:1773941619.980943:0:14694:0:(mgc_request.c:493:config_log_end()) Process entered 10000000:00000001:2.0:1773941619.980946:0:14694:0:(mgc_request.c:159:config_log_find()) Process entered 10000000:00000001:2.0:1773941619.980949:0:14694:0:(mgc_request.c:105:config_log_get()) Process entered 10000000:00000040:2.0:1773941619.980952:0:14694:0:(mgc_request.c:108:config_log_get()) log lustre-client (ffff988d181fc000) refs 2 10000000:00000001:2.0:1773941619.980956:0:14694:0:(mgc_request.c:109:config_log_get()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941619.980959:0:14694:0:(mgc_request.c:177:config_log_find()) Process leaving (rc=18446630330495385600 : -113743214166016 : ffff988d181fc000) 10000000:00000001:2.0:1773941619.981000:0:14694:0:(mgc_request.c:461:config_mark_cld_stop_nolock()) Process entered 10000000:00000040:2.0:1773941619.981004:0:14694:0:(mgc_request.c:467:config_mark_cld_stop_nolock()) lockh 0x0 00010000:00000001:2.0:1773941619.981008:0:14694:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:2.0:1773941619.981011:0:14694:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941619.981014:0:14694:0:(mgc_request.c:461:config_mark_cld_stop_nolock()) Process entered 10000000:00000040:2.0:1773941619.981019:0:14694:0:(mgc_request.c:467:config_mark_cld_stop_nolock()) lockh 0x0 00010000:00000001:2.0:1773941619.981022:0:14694:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:2.0:1773941619.981026:0:14694:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941619.981029:0:14694:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:2.0:1773941619.981031:0:14694:0:(mgc_request.c:123:config_log_put()) log lustre-cliir(ffff988d181fd400) refs 1 10000000:01000000:2.0:1773941619.981035:0:14694:0:(mgc_request.c:130:config_log_put()) dropping config log lustre-cliir 10000000:00000001:2.0:1773941619.981036:0:14694:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941619.981038:0:14694:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941619.981041:0:14694:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941619.981043:0:14694:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941619.981045:0:14694:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941619.981046:0:14694:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941619.981057:0:14694:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941619.981059:0:14694:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941619.981061:0:14694:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941619.981063:0:14694:0:(mgc_request.c:120:config_log_put()) Process leaving 00000020:00000040:2.0:1773941619.981067:0:14694:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b5800 : new refcount 5 10000000:00000010:2.0:1773941619.981071:0:14694:0:(mgc_request.c:144:config_log_put()) kfreed 'cld': 285 at ffff988d181fd400. 10000000:00000001:2.0:1773941619.981075:0:14694:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:2.0:1773941619.981078:0:14694:0:(mgc_request.c:461:config_mark_cld_stop_nolock()) Process entered 10000000:00000040:2.0:1773941619.981081:0:14694:0:(mgc_request.c:467:config_mark_cld_stop_nolock()) lockh 0x0 00010000:00000001:2.0:1773941619.981084:0:14694:0:(ldlm_lock.c:612:__ldlm_handle2lock()) Process entered 00010000:00000001:2.0:1773941619.981087:0:14694:0:(ldlm_lock.c:617:__ldlm_handle2lock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941619.981090:0:14694:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:2.0:1773941619.981093:0:14694:0:(mgc_request.c:123:config_log_put()) log params(ffff988d181fc600) refs 1 10000000:01000000:2.0:1773941619.981098:0:14694:0:(mgc_request.c:130:config_log_put()) dropping config log params 10000000:00000001:2.0:1773941619.981101:0:14694:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941619.981102:0:14694:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941619.981104:0:14694:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941619.981105:0:14694:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941619.981106:0:14694:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941619.981108:0:14694:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941619.981109:0:14694:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941619.981110:0:14694:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941619.981111:0:14694:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941619.981113:0:14694:0:(mgc_request.c:120:config_log_put()) Process leaving 00000020:00000040:2.0:1773941619.981114:0:14694:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b5800 : new refcount 4 10000000:00000010:2.0:1773941619.981117:0:14694:0:(mgc_request.c:144:config_log_put()) kfreed 'cld': 279 at ffff988d181fc600. 10000000:00000001:2.0:1773941619.981119:0:14694:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:2.0:1773941619.981121:0:14694:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941619.981122:0:14694:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941619.981123:0:14694:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:2.0:1773941619.981125:0:14694:0:(mgc_request.c:123:config_log_put()) log lustre-sptlrpc(ffff988d065c4800) refs 1 10000000:01000000:2.0:1773941619.981127:0:14694:0:(mgc_request.c:130:config_log_put()) dropping config log lustre-sptlrpc 10000000:00000001:2.0:1773941619.981129:0:14694:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941619.981130:0:14694:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941619.981131:0:14694:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941619.981132:0:14694:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941619.981134:0:14694:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941619.981135:0:14694:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941619.981136:0:14694:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941619.981137:0:14694:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941619.981145:0:14694:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941619.981146:0:14694:0:(mgc_request.c:120:config_log_put()) Process leaving 02000000:08000000:2.0:1773941619.981152:0:14694:0:(sec_config.c:500:sptlrpc_conf_free()) free sptlrpc conf lustre 02000000:00000010:2.0:1773941619.981160:0:14694:0:(sec_config.c:504:sptlrpc_conf_free()) kfreed 'conf': 120 at ffff988d181fef80. 00000020:00000040:2.0:1773941619.981166:0:14694:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b5800 : new refcount 3 10000000:00000010:2.0:1773941619.981168:0:14694:0:(mgc_request.c:144:config_log_put()) kfreed 'cld': 287 at ffff988d065c4800. 10000000:00000001:2.0:1773941619.981172:0:14694:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:2.0:1773941619.981173:0:14694:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941619.981175:0:14694:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941619.981176:0:14694:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:2.0:1773941619.981177:0:14694:0:(mgc_request.c:123:config_log_put()) log lustre-client(ffff988d181fc000) refs 2 10000000:00000001:2.0:1773941619.981180:0:14694:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:00000001:2.0:1773941619.981181:0:14694:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000040:2.0:1773941619.981183:0:14694:0:(mgc_request.c:123:config_log_put()) log lustre-client(ffff988d181fc000) refs 1 10000000:01000000:2.0:1773941619.981186:0:14694:0:(mgc_request.c:130:config_log_put()) dropping config log lustre-client 10000000:00000001:2.0:1773941619.981187:0:14694:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941619.981189:0:14694:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941619.981190:0:14694:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941619.981191:0:14694:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941619.981192:0:14694:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941619.981194:0:14694:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941619.981195:0:14694:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941619.981196:0:14694:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:00000001:2.0:1773941619.981197:0:14694:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:2.0:1773941619.981199:0:14694:0:(mgc_request.c:120:config_log_put()) Process leaving 00000020:00000040:2.0:1773941619.981200:0:14694:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b5800 : new refcount 2 10000000:00000010:2.0:1773941619.981202:0:14694:0:(mgc_request.c:144:config_log_put()) kfreed 'cld': 286 at ffff988d181fc000. 10000000:00000001:2.0:1773941619.981205:0:14694:0:(mgc_request.c:147:config_log_put()) Process leaving 10000000:01000000:2.0:1773941619.981206:0:14694:0:(mgc_request.c:547:config_log_end()) end config log lustre-client (0) 10000000:00000001:2.0:1773941619.981209:0:14694:0:(mgc_request.c:548:config_log_end()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941619.981211:0:14694:0:(mgc_request.c:2026:mgc_process_config()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1773941619.981215:0:14694:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d0381f200. 00000040:00000010:2.0:1773941619.981220:0:14694:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d0381e000. 00000020:00000010:2.0:1773941619.981223:0:14694:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941619.981227:0:14694:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d03356000. 00040000:00000010:2.0:1773941619.981230:0:14694:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d0a0db000. 00040000:00000010:2.0:1773941619.981235:0:14694:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d03357800. 00000020:00000010:2.0:1773941619.981238:0:14694:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0381e800. 00000020:00000001:2.0:1773941619.981249:0:14694:0:(obd_class.h:696:obd_process_config()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1773941619.981253:0:14694:0:(obd_mount.c:131:lustre_end_log()) kfreed 'lcfg': 176 at ffff988d09350540. 00000020:00000001:2.0:1773941619.981259:0:14694:0:(obd_mount.c:132:lustre_end_log()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941619.981263:0:14694:0:(obd_mount.c:116:lustre_end_log()) Process entered 00000020:00000010:2.0:1773941619.981269:0:14694:0:(obd_mount.c:126:lustre_end_log()) kmalloced '(lcfg)': 168 at ffff988d09350540. 00000020:00000001:2.0:1773941619.981273:0:14694:0:(obd_class.h:681:obd_process_config()) Process entered 00000020:00000010:2.0:1773941619.981277:0:14694:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d03357800. 00000020:00000010:2.0:1773941619.981283:0:14694:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d03356000. 00000040:00000010:2.0:1773941619.981288:0:14694:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d03357e00. 00000020:00000010:2.0:1773941619.981293:0:14694:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941619.981298:0:14694:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d03356600. 00040000:00000010:2.0:1773941619.981304:0:14694:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d0a0db000. 00040000:00000010:2.0:1773941619.981309:0:14694:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d03356400. 10000000:00000001:2.0:1773941619.981315:0:14694:0:(mgc_request.c:1932:mgc_process_config()) Process entered 10000000:00000001:2.0:1773941619.981318:0:14694:0:(mgc_request_server.c:668:mgc_process_config_server()) Process entered 10000000:00000001:2.0:1773941619.981322:0:14694:0:(mgc_request.c:493:config_log_end()) Process entered 10000000:00000001:2.0:1773941619.981325:0:14694:0:(mgc_request.c:159:config_log_find()) Process entered 10000000:00000001:2.0:1773941619.981328:0:14694:0:(mgc_request.c:177:config_log_find()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941619.981332:0:14694:0:(mgc_request.c:497:config_log_end()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 10000000:00000001:2.0:1773941619.981339:0:14694:0:(mgc_request.c:2026:mgc_process_config()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000020:00000010:2.0:1773941619.981347:0:14694:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d03356000. 00000040:00000010:2.0:1773941619.981353:0:14694:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d03357e00. 00000020:00000010:2.0:1773941619.981363:0:14694:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941619.981369:0:14694:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d03356600. 00040000:00000010:2.0:1773941619.981377:0:14694:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d0a0db000. 00040000:00000010:2.0:1773941619.981382:0:14694:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d03356400. 00000020:00000010:2.0:1773941619.981387:0:14694:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d03357800. 00000020:00000001:2.0:1773941619.981391:0:14694:0:(obd_class.h:696:obd_process_config()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000020:00000010:2.0:1773941619.981396:0:14694:0:(obd_mount.c:131:lustre_end_log()) kfreed 'lcfg': 168 at ffff988d09350540. 00000020:00000001:2.0:1773941619.981400:0:14694:0:(obd_mount.c:132:lustre_end_log()) Process leaving (rc=18446744073709551614 : -2 : fffffffffffffffe) 00000080:02000400:2.0:1773941619.981409:0:14694:0:(llite_lib.c:1617:ll_put_super()) Unmounted lustre-client 00000100:00080000:3.0:1773941619.988333:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to EVICTED 00000100:00000001:3.0:1773941619.988341:0:2400:0:(import.c:1283:ptlrpc_connect_interpret()) Process leaving via finish (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941619.988347:0:2400:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000100:00080000:3.0:1773941619.988355:0:2400:0:(import.c:1660:ptlrpc_import_recovery_state_machine()) evicted from MGS@192.168.204.114@tcp; invalidating 00000020:00000040:3.0:1773941619.988360:0:2400:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=5 obd=MGC192.168.204.114@tcp 00000100:00000001:3.0:1773941619.988737:0:2400:0:(import.c:1682:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941619.988744:0:2400:0:(import.c:1505:ptlrpc_connect_interpret()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:3.0:1773941619.988758:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d10836a00 x1860112291500672/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941629 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941619.988781:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d10836a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291500672:192.168.204.114@tcp:250:ll_cfg_requeue.0 00000100:00000001:0.0:1773941619.988781:0:14820:0:(import.c:1580:ptlrpc_invalidate_import_thread()) Process entered 00000100:00080000:0.0:1773941619.988809:0:14820:0:(import.c:1584:ptlrpc_invalidate_import_thread()) thread invalidate import MGC192.168.204.114@tcp to MGS@192.168.204.114@tcp 00000100:00000001:3.0:1773941619.988811:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1773941619.988822:0:14820:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:3.0:1773941619.988823:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1773941619.988838:0:14820:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000040:3.0:1773941619.988846:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d10836a00 x1860112291500672/t0(0) o250->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 520/416 e 0 to 0 dl 1773941629 ref 1 fl Complete:RNQU/200/0 rc 0/0 job:'ll_cfg_requeue.0' uid:0 gid:0 projid:4294967295 10000000:01000000:0.0:1773941619.988847:0:14820:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:0.0:1773941619.988862:0:14820:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941619.988872:0:14820:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:3.0:1773941619.988881:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 00000001:00000001:0.0:1773941619.988883:0:14820:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941619.988891:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000001:00000001:0.0:1773941619.988893:0:14820:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:0.0:1773941619.988906:0:14820:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 02000000:00000010:3.0:1773941619.988910:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 1024 at ffff988d03522400. 00000001:00000001:0.0:1773941619.988912:0:14820:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:0.0:1773941619.988947:0:14820:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941619.988955:0:14820:0:(obd_class.h:1226:obd_import_event()) Process leaving 02000000:00000001:3.0:1773941619.988957:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:0.0:1773941619.989010:0:14820:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from EVICTED to RECOVER 00000100:00000001:0.0:1773941619.989017:0:14820:0:(import.c:1634:ptlrpc_import_recovery_state_machine()) Process entered 00000020:00000001:3.0:1773941619.989020:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000100:00000001:0.0:1773941619.989023:0:14820:0:(recover.c:147:ptlrpc_resend()) Process entered 00000020:00000040:3.0:1773941619.989026:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=4 obd=MGC192.168.204.114@tcp 00000100:00000001:0.0:1773941619.989029:0:14820:0:(recover.c:176:ptlrpc_resend()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941619.989036:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00080000:0.0:1773941619.989039:0:14820:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from RECOVER to FULL 02000000:00000010:3.0:1773941619.989042:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 1024 at ffff988d02ce7000. 00000100:00000001:0.0:1773941619.989056:0:14820:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:3.0:1773941619.989058:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 10000000:01000000:0.0:1773941619.989062:0:14820:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808004 02000000:00000001:3.0:1773941619.989063:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 00000100:00000010:3.0:1773941619.989077:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d10836a00. 10000000:00000040:0.0:1773941619.989079:0:14820:0:(mgc_request.c:1183:mgc_import_event()) MGC192.168.204.114@tcp: Reactivating import 00000100:00000001:3.0:1773941619.989096:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1773941619.989104:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941619.989109:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1773941619.989110:0:14820:0:(pinger.c:185:ptlrpc_pinger_ir_up()) IR up 10000000:00000001:0.0:1773941619.989116:0:14820:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941619.989120:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1773941619.989121:0:14820:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941619.989128:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941619.989135:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:02000000:0.0:1773941619.989140:0:14820:0:(import.c:1730:ptlrpc_import_recovery_state_machine()) MGC192.168.204.114@tcp: Connection restored to 192.168.204.114@tcp (at 192.168.204.114@tcp) 00000100:00000001:3.0:1773941619.989141:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1773941619.989157:0:14820:0:(import.c:1742:ptlrpc_import_recovery_state_machine()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941619.989162:0:14820:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:0.0:1773941619.989166:0:14820:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:0.0:1773941619.989170:0:14820:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:0.0:1773941619.989173:0:14820:0:(import.c:1597:ptlrpc_invalidate_import_thread()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941619.990869:0:14694:0:(obd_config.c:1231:class_del_profile()) Process entered 00000020:01000000:2.0:1773941619.990874:0:14694:0:(obd_config.c:1233:class_del_profile()) Del profile lustre-client 00000020:00000001:2.0:1773941619.990876:0:14694:0:(obd_config.c:1149:class_get_profile_nolock()) Process entered 00000020:00000001:2.0:1773941619.990891:0:14694:0:(obd_config.c:1156:class_get_profile_nolock()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941619.990895:0:14694:0:(obd_config.c:1248:class_del_profile()) Process leaving 00000080:00000001:2.0:1773941619.990899:0:14694:0:(llite_lib.c:266:ll_free_sbi()) Process entered 00000020:00000010:2.0:1773941619.990953:0:14694:0:(cl_page.c:1415:cl_cache_decref()) kfreed 'cache': 152 at ffff988d02eb6e40. 00000080:00000010:2.0:1773941619.991013:0:14694:0:(llite_lib.c:281:ll_free_sbi()) kfreed 'sbi->ll_foreign_symlink_prefix': 6 at ffff988d0723df88. 00000080:00000010:2.0:1773941619.991022:0:14694:0:(llite_lib.c:287:ll_free_sbi()) kfreed 'sbi->ll_foreign_symlink_upcall': 5 at ffff988d0723d4b8. 00000080:00000010:2.0:1773941619.991071:0:14694:0:(llite_lib.c:310:ll_free_sbi()) kfreed 'sbi': 1376 at ffff988d111b6800. 00000080:00000001:2.0:1773941619.991077:0:14694:0:(llite_lib.c:312:ll_free_sbi()) Process leaving 00000020:00000001:2.0:1773941619.991082:0:14694:0:(obd_mount.c:909:lustre_common_put_super()) Process entered 00000020:01000004:2.0:1773941619.991085:0:14694:0:(obd_mount.c:911:lustre_common_put_super()) dropping sb ffff988d111b1800 00000020:00000001:2.0:1773941619.991090:0:14694:0:(obd_mount.c:575:lustre_stop_mgc()) Process entered 00000020:00000001:2.0:1773941619.991095:0:14694:0:(obd_class.h:902:obd_disconnect()) Process entered 00010000:00000001:2.0:1773941619.991102:0:14694:0:(ldlm_lib.c:759:client_disconnect_export()) Process entered 00010000:00000040:2.0:1773941619.991106:0:14694:0:(ldlm_lib.c:772:client_disconnect_export()) disconnect MGC192.168.204.114@tcp - 1 00000100:00000001:2.0:1773941619.991113:0:14694:0:(pinger.c:388:ptlrpc_pinger_del_import()) Process entered 00000100:00080000:2.0:1773941619.991119:0:14694:0:(pinger.c:396:ptlrpc_pinger_del_import()) removing pingable import c1c1ff21-5f68-4bf6-bdc4-258644b24465->MGS 00000020:00000001:2.0:1773941619.991123:0:14694:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941619.991128:0:14694:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=2 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941619.991133:0:14694:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941619.991136:0:14694:0:(pinger.c:401:ptlrpc_pinger_del_import()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941619.991142:0:14694:0:(ldlm_request.c:2367:ldlm_cli_cancel_unused()) Process entered 00000001:00000001:2.0:1773941619.991147:0:14694:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941619.991151:0:14694:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941619.991158:0:14694:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941619.991163:0:14694:0:(ldlm_request.c:2379:ldlm_cli_cancel_unused()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941619.991168:0:14694:0:(ldlm_resource.c:1268:ldlm_namespace_free_prior()) Process entered 00010000:00000001:2.0:1773941619.991173:0:14694:0:(ldlm_resource.c:1207:__ldlm_namespace_free()) Process entered 00000001:00000001:2.0:1773941619.991176:0:14694:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941619.991178:0:14694:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941619.991181:0:14694:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:2.0:1773941619.991184:0:14694:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941619.991186:0:14694:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941619.991189:0:14694:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941619.991194:0:14694:0:(ldlm_resource.c:1251:__ldlm_namespace_free()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941619.991197:0:14694:0:(ldlm_resource.c:1293:ldlm_namespace_free_prior()) Process leaving 00000100:00000001:2.0:1773941619.991201:0:14694:0:(import.c:1924:ptlrpc_disconnect_import()) Process entered 00000100:00000001:2.0:1773941619.991206:0:14694:0:(import.c:1858:ptlrpc_disconnect_import_async()) Process entered 00000100:00000001:2.0:1773941619.991223:0:14694:0:(import.c:1750:ptlrpc_disconnect_prep_req()) Process entered 00000100:00000010:2.0:1773941619.991231:0:14694:0:(client.c:592:ptlrpc_request_cache_alloc()) slab-alloced 'req': 888 at ffff988d06506a00. 00000020:00000040:2.0:1773941619.991238:0:14694:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941619.991244:0:14694:0:(client.c:844:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1773941619.991248:0:14694:0:(sec.c:440:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1773941619.991255:0:14694:0:(sec.c:464:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:2.0:1773941619.991264:0:14694:0:(sec_null.c:146:null_alloc_reqbuf()) kmalloced '(req->rq_reqbuf)': 256 at ffff988d0a0db000. 00000100:00000001:2.0:1773941619.991272:0:14694:0:(client.c:924:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941619.991278:0:14694:0:(import.c:1788:ptlrpc_disconnect_prep_req()) Process leaving (rc=18446630330196584960 : -113743512966656 : ffff988d06506a00) 00000100:00080000:2.0:1773941619.991285:0:14694:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from FULL to CONNECTING 00000100:00000001:2.0:1773941619.991293:0:14694:0:(jobid.c:936:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1773941619.991297:0:14694:0:(jobid.c:976:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000040:2.0:1773941619.991307:0:14694:0:(ptlrpcd.c:235:ptlrpcd_add_req()) @@@ add req [00000000c6b603d8] to pc [ptlrpcd_rcv+-1] req@ffff988d06506a00 x1860112291500800/t0(0) o251->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1773941619.991338:0:14694:0:(import.c:1904:ptlrpc_disconnect_import_async()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941619.991374:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941619.991379:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941619.991382:0:2400:0:(client.c:1797:ptlrpc_send_new_req()) Process entered 00000100:00000040:3.0:1773941619.991391:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from New to Rpc req@ffff988d06506a00 x1860112291500800/t0(0) o251->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 0 ref 1 fl New:NQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941619.991404:0:2400:0:(client.c:1363:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1773941619.991406:0:2400:0:(client.c:1421:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941619.991410:0:2400:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1773941619.991413:0:2400:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1773941619.991416:0:2400:0:(sec.c:774:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1773941619.991420:0:2400:0:(client.c:1903:ptlrpc_send_new_req()) Sending RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291500800:192.168.204.114@tcp:251:mount.lustre.0 00000100:00000001:3.0:1773941619.991426:0:2400:0:(niobuf.c:865:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1773941619.991439:0:2400:0:(sec.c:1068:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1773941619.991442:0:2400:0:(sec.c:1105:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941619.991445:0:2400:0:(sec.c:1852:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000010:3.0:1773941619.991450:0:2400:0:(sec_null.c:189:null_alloc_repbuf()) kmalloced '(req->rq_repbuf)': 512 at ffff988d1a1c8800. 02000000:00000001:3.0:1773941619.991453:0:2400:0:(sec.c:1862:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941619.991460:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000010:3.0:1773941619.991465:0:2400:0:(lib-me.c:70:LNetMEAttach()) slab-alloced 'me': 88 at ffff988d03ccd0b0. 00000400:00000010:3.0:1773941619.991470:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182acee0. 00000100:00000200:3.0:1773941619.991476:0:2400:0:(niobuf.c:1096:ptl_send_rpc()) Setup reply buffer: 512 bytes, xid 1860112291500800, portal 25 00000100:00000001:3.0:1773941619.991479:0:2400:0:(client.c:3244:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1773941619.991481:0:2400:0:(client.c:3246:ptlrpc_request_addref()) Process leaving (rc=18446630330196584960 : -113743512966656 : ffff988d06506a00) 00000100:00000040:3.0:1773941619.991489:0:2400:0:(niobuf.c:1116:ptl_send_rpc()) @@@ send flags=200 req@ffff988d06506a00 x1860112291500800/t0(0) o251->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941629 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941619.991498:0:2400:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000040:3.0:1773941619.991502:0:2400:0:(niobuf.c:72:ptl_send_buf()) peer_id 12345-192.168.204.114@tcp 00000400:00000010:3.0:1773941619.991506:0:2400:0:(lib-md.c:131:lnet_md_build()) slab-alloced 'lmd': 136 at ffff988d182aca18. 00000100:00000200:3.0:1773941619.991509:0:2400:0:(niobuf.c:94:ptl_send_buf()) Sending 224 bytes to portal 26, xid 1860112291500800, offset 0 00000400:00000200:3.0:1773941619.991516:0:2400:0:(lib-move.c:5316:LNetPut()) LNetPut -> 12345-192.168.204.114@tcp 00000400:00000200:3.0:1773941619.991523:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:3.0:1773941619.991531:0:2400:0:(lib-move.c:2973:lnet_handle_send_case_locked()) Source ANY to MR: 192.168.204.114@tcp local destination 00000400:00000200:3.0:1773941619.991535:0:2400:0:(lib-move.c:2603:lnet_find_best_ni_on_local_net()) selected best_lpn tcp 00000400:00000200:3.0:1773941619.991540:0:2400:0:(lib-move.c:1789:lnet_get_best_ni()) selected best_ni 192.168.204.14@tcp 00000400:00000200:3.0:1773941619.991544:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941619.991549:0:2400:0:(lib-move.c:1343:lnet_select_peer_ni()) 192.168.204.14@tcp lpni_is_preferred = 0 00000400:00000200:3.0:1773941619.991556:0:2400:0:(lib-move.c:1358:lnet_select_peer_ni()) n:[192.168.204.114@tcp, 192.168.204.124@tcp] h:[1000, 1000] p:[-1, -1] c:[8, 8] s:[190, 189] 00000400:00000200:3.0:1773941619.991565:0:2400:0:(lib-move.c:1412:lnet_select_peer_ni()) sd_best_lpni = 192.168.204.124@tcp 00000400:00000200:3.0:1773941619.991574:0:2400:0:(lib-move.c:1909:lnet_handle_send()) 192.168.204.14@tcp NI seq info: [301:301:256:4294967295] 192.168.204.124@tcp LPNI seq info [191:191:8:4294967295] 00000400:00000200:3.0:1773941619.991583:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:3.0:1773941619.991602:0:2400:0:(lib-move.c:2024:lnet_handle_send()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp:) -> 192.168.204.124@tcp(192.168.204.114@tcp:192.168.204.124@tcp) : PUT try# 0 00000800:00000200:3.0:1773941619.991613:0:2400:0:(socklnd_cb.c:1001:ksocknal_send()) sending 224 bytes in 1 frags to 12345-192.168.204.124@tcp 00000800:00000010:3.0:1773941619.991621:0:2400:0:(socklnd_cb.c:41:ksocknal_alloc_tx()) alloc '(tx)': 248 at ffff988d08f74c00. 00000800:00000200:3.0:1773941619.991632:0:2400:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [ffff988d1837d6c0] -> 12345-192.168.204.124@tcp (5) 00000800:00000200:3.0:1773941619.991653:0:2400:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.204.124@tcp ip 192.168.204.124:988 00000800:00000200:3.0:1773941619.991660:0:2400:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet ffff988d08f74c00 type 1, nob 320 niov 1 nkiov 1 00000100:00000001:3.0:1773941619.991685:0:2400:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000400:00000200:3.0:1773941619.991691:0:2400:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000100:00000001:3.0:1773941619.991695:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 0 is 4295011587 00000100:00000001:3.0:1773941619.991701:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 0, simply extend its deadline from 4295011565 00000100:00000001:3.0:1773941619.991706:0:2400:0:(niobuf.c:662:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS new deadline estimation for cpu 1 is 4295011587 00000100:00000001:3.0:1773941619.991710:0:2400:0:(niobuf.c:684:kick_cpu_latency()) MGC192.168.204.114@tcp: PM QoS request active for cpu 1, simply extend its deadline from 4295011565 00000100:00000001:3.0:1773941619.991714:0:2400:0:(niobuf.c:1134:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1773941619.991718:0:2400:0:(client.c:1925:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941619.991721:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941619.991724:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941619.991729:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1773941619.991731:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1773941619.991734:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1773941619.991736:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1773941619.991738:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941619.991740:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000800:00000200:1.0:1773941619.991890:0:2394:0:(socklnd_cb.c:539:ksocknal_process_transmit()) send(0) 0 00000800:00000010:1.0:1773941619.991909:0:2394:0:(socklnd_cb.c:98:ksocknal_free_tx()) kfreed 'tx': 248 at ffff988d08f74c00. 00000400:00000200:1.0:1773941619.991916:0:2394:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941619.991955:0:2394:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000400:00000200:1.0:1773941619.991990:0:2394:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182aca18 00000400:00000010:1.0:1773941619.991995:0:2394:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182aca18. 00000100:00000001:1.0:1773941619.992005:0:2394:0:(events.c:34:request_out_callback()) Process entered 00000100:00000200:1.0:1773941619.992017:0:2394:0:(events.c:39:request_out_callback()) @@@ type 5, status 0 req@ffff988d06506a00 x1860112291500800/t0(0) o251->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941629 ref 2 fl Rpc:Nr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941619.992032:0:2394:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:1.0:1773941619.992038:0:2394:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 1 req@ffff988d06506a00 x1860112291500800/t0(0) o251->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941629 ref 2 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941619.992049:0:2394:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1773941619.992051:0:2394:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000010:2.1:1773941619.992999:0:0:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d089de400. 00000800:00000001:1.0:1773941619.993740:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941619.993759:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000800:00000001:1.0:1773941619.993764:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941619.993767:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941619.993775:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941619.993782:0:2393:0:(lib-move.c:4806:lnet_parse()) TRACE: 192.168.204.14@tcp(192.168.204.14@tcp) <- 192.168.204.124@tcp : PUT - for me 00000400:00000200:1.0:1773941619.993791:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941619.993797:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.124@tcp to cpt 0 00000400:00000200:1.0:1773941619.993803:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941619.993808:0:2393:0:(lib-ptl.c:584:lnet_ptl_match_md()) Request from 12345-192.168.204.114@tcp of length 224 into portal 25 MB=0x69bc324609300 00000400:00000200:1.0:1773941619.993814:0:2393:0:(api-ni.c:1560:lnet_nid4_cpt_hash()) Match nid 192.168.204.114@tcp to cpt 0 00000400:00000200:1.0:1773941619.993821:0:2393:0:(lib-ptl.c:210:lnet_try_match_md()) Incoming put index 19 from 12345-192.168.204.114@tcp of length 224/224 into md 0x12f1 [1] + 224 00000800:00000001:1.0:1773941619.993829:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000800:00000001:1.0:1773941619.993840:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000200:1.0:1773941619.993843:0:2393:0:(lib-msg.c:1048:lnet_is_health_check()) health check = 1, status = 0, hstatus = 0 00000400:00000200:1.0:1773941619.993848:0:2393:0:(lib-msg.c:824:lnet_health_check()) health check: 192.168.204.14@tcp->192.168.204.124@tcp: PUT: OK 00000100:00000001:1.0:1773941619.993854:0:2393:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:1.0:1773941619.993862:0:2393:0:(events.c:80:reply_in_callback()) @@@ type 2, status 0 req@ffff988d06506a00 x1860112291500800/t0(0) o251->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941629 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000040:1.0:1773941619.993887:0:2393:0:(events.c:149:reply_in_callback()) @@@ reply in flags=200 mlen=224 offset=224 replen=224 req@ffff988d06506a00 x1860112291500800/t0(0) o251->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941629 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1773941619.993951:0:2393:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941619.994002:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000001:1.0:1773941619.994009:0:2393:0:(socklnd_cb.c:345:ksocknal_receive()) Process entered 00000100:00000001:3.0:1773941619.994010:0:2400:0:(client.c:1974:ptlrpc_check_set()) Process entered 00000800:00000001:1.0:1773941619.994021:0:2393:0:(socklnd_cb.c:383:ksocknal_receive()) Process leaving (rc=1 : 1 : 1) 00000400:00000001:3.0:1773941619.994023:0:2400:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941619.994031:0:2400:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941619.994036:0:2400:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccd0b0. 00000400:00000200:3.0:1773941619.994044:0:2400:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d182acee0 00000400:00000010:3.0:1773941619.994048:0:2400:0:(lib-lnet.h:312:lnet_md_free()) slab-freed 'md': 136 at ffff988d182acee0. 00000100:00000001:3.0:1773941619.994053:0:2400:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000200:3.0:1773941619.994063:0:2400:0:(events.c:80:reply_in_callback()) @@@ type 6, status 0 req@ffff988d06506a00 x1860112291500800/t0(0) o251->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941629 ref 1 fl Rpc:RNQ/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000200:3.0:1773941619.994077:0:2400:0:(events.c:102:reply_in_callback()) @@@ unlink req@ffff988d06506a00 x1860112291500800/t0(0) o251->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941629 ref 1 fl Rpc:RNQU/200/ffffffff rc 0/-1 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941619.994088:0:2400:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1773941619.994090:0:2400:0:(client.c:2957:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941619.994092:0:2400:0:(client.c:1578:after_reply()) Process entered 02000000:00000001:3.0:1773941619.994106:0:2400:0:(sec.c:1113:do_cli_unwrap_reply()) Process entered 00000100:00000001:3.0:1773941619.994109:0:2400:0:(pack_generic.c:602:__lustre_unpack_msg()) Process entered 00000100:00000001:3.0:1773941619.994112:0:2400:0:(pack_generic.c:620:__lustre_unpack_msg()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1773941619.994115:0:2400:0:(sec.c:1169:do_cli_unwrap_reply()) Process leaving (rc=0 : 0 : 0) 00000100:00001000:3.0:1773941619.994121:0:2400:0:(import.c:2136:obd_at_measure()) add 0 to ffff988d20336458 time=120 v=5 (5 0 0 0) 00000100:00001000:3.0:1773941619.994125:0:2400:0:(import.c:2136:obd_at_measure()) add 1 to ffff988d20336410 time=0 v=5 (1 0 0 0) 00000100:00000001:3.0:1773941619.994129:0:2400:0:(client.c:1480:ptlrpc_check_status()) Process entered 00000100:00000001:3.0:1773941619.994131:0:2400:0:(client.c:1491:ptlrpc_check_status()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000001:3.0:1773941619.994135:0:2400:0:(client.c:1695:after_reply()) Process leaving (rc=18446744073709551509 : -107 : ffffffffffffff95) 00000100:00000040:3.0:1773941619.994142:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Rpc to Interpret req@ffff988d06506a00 x1860112291500800/t0(0) o251->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941629 ref 1 fl Rpc:RNQU/200/0 rc -107/-107 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941619.994151:0:2400:0:(client.c:2329:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551509 : -107 : 0xffffffffffffff95) 00000100:00000001:3.0:1773941619.994154:0:2400:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1773941619.994156:0:2400:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1773941619.994160:0:2400:0:(import.c:65:import_set_state_nolock()) ffff988d20336000 MGS: changing import state from CONNECTING to CLOSED 00000100:00000001:3.0:1773941619.994163:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941619.994166:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:3.0:1773941619.994169:0:2400:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:00000001:3.0:1773941619.994171:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941619.994173:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941619.994175:0:2400:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:3.0:1773941619.994177:0:2400:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:3.0:1773941619.994179:0:2400:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941619.994181:0:2400:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000040:3.0:1773941619.994200:0:2400:0:(lustre_net.h:2431:ptlrpc_rqphase_move()) @@@ move request phase from Interpret to Complete req@ffff988d06506a00 x1860112291500800/t0(0) o251->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941629 ref 1 fl Interpret:RNQU/200/0 rc 0/-107 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1773941619.994211:0:2400:0:(client.c:2389:ptlrpc_check_set()) Completed RPC req@ffff988d06506a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:c1c1ff21-5f68-4bf6-bdc4-258644b24465:2400:1860112291500800:192.168.204.114@tcp:251:mount.lustre.0 00000100:00000001:3.0:1773941619.994218:0:2400:0:(client.c:2440:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1773941619.994221:0:2400:0:(client.c:2842:__ptlrpc_req_put()) Process entered 00000100:00000040:3.0:1773941619.994225:0:2400:0:(client.c:2850:__ptlrpc_req_put()) @@@ refcount now 0 req@ffff988d06506a00 x1860112291500800/t0(0) o251->MGC192.168.204.114@tcp@192.168.204.114@tcp:26/25 lens 224/224 e 0 to 0 dl 1773941629 ref 1 fl Complete:RNQU/200/0 rc 0/-107 job:'mount.lustre.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1773941619.994234:0:2400:0:(client.c:2768:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1773941619.994244:0:2400:0:(sec.c:1877:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1773941619.994245:0:14694:0:(import.c:1967:ptlrpc_disconnect_import()) Process leaving (rc=0 : 0 : 0) 02000000:00000010:3.0:1773941619.994248:0:2400:0:(sec_null.c:203:null_free_repbuf()) kfreed 'req->rq_repbuf': 512 at ffff988d1a1c8800. 00000100:00000001:2.0:1773941619.994257:0:14694:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 02000000:00000001:3.0:1773941619.994258:0:2400:0:(sec.c:1891:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:2.0:1773941619.994265:0:14694:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000020:00000001:3.0:1773941619.994266:0:2400:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:3.0:1773941619.994271:0:2400:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=2 obd=MGC192.168.204.114@tcp 00000100:00000001:2.0:1773941619.994274:0:14694:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000020:00000001:3.0:1773941619.994277:0:2400:0:(genops.c:1249:class_import_put()) Process leaving 00000100:00000001:2.0:1773941619.994279:0:14694:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 02000000:00000010:3.0:1773941619.994283:0:2400:0:(sec_null.c:173:null_free_reqbuf()) kvfree 'req->rq_reqbuf': 256 at ffff988d0a0db000. 00000100:00000001:2.0:1773941619.994285:0:14694:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941619.994293:0:14694:0:(obd_class.h:1215:obd_import_event()) Process entered 02000000:00000001:3.0:1773941619.994294:0:2400:0:(sec.c:480:sptlrpc_req_put_ctx()) Process entered 10000000:01000000:2.0:1773941619.994298:0:14694:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 02000000:00000001:3.0:1773941619.994300:0:2400:0:(sec.c:497:sptlrpc_req_put_ctx()) Process leaving 10000000:00000001:2.0:1773941619.994306:0:14694:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000010:3.0:1773941619.994307:0:2400:0:(client.c:598:ptlrpc_request_cache_free()) slab-freed '(req)': 888 at ffff988d06506a00. 00000100:00000001:2.0:1773941619.994316:0:14694:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00000001:3.0:1773941619.994318:0:2400:0:(client.c:2824:__ptlrpc_free_req()) Process leaving 00000100:00100000:2.0:1773941619.994322:0:14694:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:3.0:1773941619.994323:0:2400:0:(client.c:2883:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1773941619.994328:0:14694:0:(obd_class.h:1215:obd_import_event()) Process entered 00000100:00000001:3.0:1773941619.994331:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 10000000:01000000:2.0:1773941619.994333:0:14694:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:2.0:1773941619.994339:0:14694:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:3.0:1773941619.994341:0:2400:0:(client.c:2614:ptlrpc_set_next_timeout()) Process entered 00000001:00000001:2.0:1773941619.994342:0:14694:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000100:00000001:3.0:1773941619.994347:0:2400:0:(client.c:2642:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:2.0:1773941619.994350:0:14694:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941619.994353:0:2400:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000001:00000001:2.0:1773941619.994356:0:14694:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000100:00000001:3.0:1773941619.994359:0:2400:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:2.0:1773941619.994361:0:14694:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941619.994368:0:14694:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941619.994371:0:14694:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941619.994373:0:14694:0:(obd_class.h:1226:obd_import_event()) Process leaving 00010000:00000001:2.0:1773941619.994393:0:14694:0:(ldlm_lib.c:818:client_disconnect_export()) Process leaving 00000020:00000001:2.0:1773941619.994396:0:14694:0:(genops.c:1462:class_disconnect()) Process entered 00000020:00000080:2.0:1773941619.994400:0:14694:0:(genops.c:1487:class_disconnect()) disconnect: cookie 0x8e86767622e28d4 00000020:00000040:2.0:1773941619.994405:0:14694:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d111b0800 with handle 0x8e86767622e28d4 from hash 00000020:00000001:2.0:1773941619.994412:0:14694:0:(genops.c:1934:obd_stale_export_put()) Process entered 00000020:00000040:2.0:1773941619.994415:0:14694:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 1 00000020:00000001:2.0:1773941619.994418:0:14694:0:(genops.c:1959:obd_stale_export_put()) Process leaving 00000020:00000040:2.0:1773941619.994420:0:14694:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b0800 : new refcount 0 00000020:00000080:2.0:1773941619.994423:0:14694:0:(genops.c:904:class_export_put()) final put ffff988d111b0800/c1c1ff21-5f68-4bf6-bdc4-258644b24465 00000020:00000001:2.0:1773941619.994427:0:14694:0:(lprocfs_status_server.c:686:lprocfs_exp_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941619.994449:0:14694:0:(genops.c:1493:class_disconnect()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1773941619.994453:0:14694:0:(ldlm_lib.c:831:client_disconnect_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941619.994455:0:14694:0:(obd_class.h:914:obd_disconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941619.994458:0:14694:0:(obd_config.c:2489:class_manual_cleanup()) Process entered 00000020:01000000:2.0:1773941619.994461:0:14694:0:(obd_config.c:2502:class_manual_cleanup()) Manual cleanup of MGC192.168.204.114@tcp (flags='') 00000020:00000010:2.0:1773941619.994468:0:14694:0:(obd_config.c:2506:class_manual_cleanup()) kmalloced '(lcfg)': 72 at ffff988d08bbb8a0. 00000020:00000080:2.0:1773941619.994471:0:14694:0:(obd_config.c:1506:class_process_config()) processing cmd: cf004 00000020:00000001:2.0:1773941619.994477:0:14694:0:(obd_config.c:936:class_cleanup()) Process entered 00000020:00000080:2.0:1773941619.994481:0:14694:0:(obd_config.c:991:class_cleanup()) MGC192.168.204.114@tcp: forcing exports to disconnect: 0/3 00000020:00000001:2.0:1773941619.994485:0:14694:0:(genops.c:1562:class_disconnect_exports()) Process entered 00000020:00080000:2.0:1773941619.994488:0:14694:0:(genops.c:1577:class_disconnect_exports()) OBD device 0 (ffff988d2039cd28) has no exports 00000020:00000001:2.0:1773941619.994491:0:14694:0:(genops.c:1578:class_disconnect_exports()) Process leaving 00000020:00000001:2.0:1773941619.994496:0:14694:0:(obd_class.h:604:obd_precleanup()) Process entered 00000020:00000010:2.0:1773941619.994500:0:14694:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d03357800. 00000020:00000010:2.0:1773941619.994505:0:14694:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d03356400. 00000040:00000010:2.0:1773941619.994509:0:14694:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d03356600. 00000020:00000010:2.0:1773941619.994513:0:14694:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941619.994517:0:14694:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d03357e00. 00040000:00000010:2.0:1773941619.994523:0:14694:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d0a0db600. 00040000:00000010:2.0:1773941619.994528:0:14694:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d03356000. 00000020:00000010:2.0:1773941619.994535:0:14694:0:(lu_object.c:2003:lu_env_add_task()) kmalloced '(lei)': 40 at ffff988d054378c0. 10000000:00000001:2.0:1773941619.994540:0:14694:0:(mgc_request.c:792:mgc_device_fini()) Process entered 00000020:00000001:0.0:1773941619.994551:0:36:0:(genops.c:857:class_export_destroy()) Process entered 00000020:00000080:0.0:1773941619.994559:0:36:0:(genops.c:863:class_export_destroy()) destroying export ffff988d111b0800/c1c1ff21-5f68-4bf6-bdc4-258644b24465 for MGC192.168.204.114@tcp 00000020:00000040:0.0:1773941619.994568:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 2 to 192.168.204.114@tcp 00000020:00000001:0.0:1773941619.994573:0:36:0:(obd_class.h:1017:obd_destroy_export()) Process entered 00000020:00000001:0.0:1773941619.994597:0:36:0:(obd_class.h:1021:obd_destroy_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000040:0.0:1773941619.994603:0:36:0:(obd_config.c:1058:class_decref()) Decref MGC192.168.204.114@tcp (ffff988d2039cd28) now 5 - export 00000020:00000010:0.0:1773941619.994609:0:36:0:(genops.c:879:class_export_destroy()) kfreed_rcu 'exp': 1208 at ffff988d111b0800. 00000020:00000001:0.0:1773941619.994636:0:36:0:(genops.c:880:class_export_destroy()) Process leaving 10000000:00000001:3.0:1773941619.994637:0:14708:0:(mgc_request.c:117:config_log_put()) Process entered 10000000:00000001:3.0:1773941619.994645:0:14708:0:(mgc_request.c:120:config_log_put()) Process leaving 10000000:01000000:3.0:1773941619.994660:0:14708:0:(mgc_request.c:708:mgc_requeue_thread()) Ending requeue thread 10000000:00000001:3.0:1773941619.994663:0:14708:0:(mgc_request.c:709:mgc_requeue_thread()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941619.994696:0:14694:0:(obd_class.h:652:obd_cleanup_client_import()) Process entered 10000000:01000000:2.0:1773941619.994701:0:14694:0:(obd_class.h:663:obd_cleanup_client_import()) MGC192.168.204.114@tcp: client import never connected 00000100:00000001:2.0:1773941619.994704:0:14694:0:(import.c:135:ptlrpc_deactivate_import_nolock()) Process entered 00000100:00080000:2.0:1773941619.994719:0:14694:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00000001:2.0:1773941619.994722:0:14694:0:(client.c:3544:ptlrpc_abort_inflight()) Process entered 00000100:00000001:2.0:1773941619.994725:0:14694:0:(client.c:3589:ptlrpc_abort_inflight()) Process leaving 00000100:00000001:2.0:1773941619.994727:0:14694:0:(import.c:144:ptlrpc_deactivate_import_nolock()) Process leaving 00000100:00000001:2.0:1773941619.994730:0:14694:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941619.994733:0:14694:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 10000000:00000001:2.0:1773941619.994736:0:14694:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941619.994739:0:14694:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000100:00100000:2.0:1773941619.994742:0:14694:0:(import.c:321:ptlrpc_invalidate_import()) Sleeping 20s for inflight to error out 00000100:00000001:2.0:1773941619.994745:0:14694:0:(obd_class.h:1215:obd_import_event()) Process entered 10000000:01000000:2.0:1773941619.994747:0:14694:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808003 00000001:00000001:2.0:1773941619.994750:0:14694:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941619.994753:0:14694:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941619.994757:0:14694:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 00000001:00000001:2.0:1773941619.994759:0:14694:0:(hash.c:1696:cfs_hash_for_each_nolock()) Process entered 00000001:00000001:2.0:1773941619.994761:0:14694:0:(hash.c:1609:cfs_hash_for_each_relax()) Process entered 00000001:00000001:2.0:1773941619.994763:0:14694:0:(hash.c:1712:cfs_hash_for_each_nolock()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941619.994765:0:14694:0:(mgc_request.c:1199:mgc_import_event()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941619.994768:0:14694:0:(obd_class.h:1226:obd_import_event()) Process leaving 00000020:00000040:2.0:1773941619.994772:0:14694:0:(genops.c:1229:class_import_get()) import ffff988d20336000 refcount=3 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941619.994775:0:14694:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941619.994778:0:14694:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=2 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941619.994781:0:14694:0:(genops.c:1249:class_import_put()) Process leaving 02000000:08000000:2.0:1773941619.994785:0:14694:0:(sec.c:1366:sec_cop_destroy_sec()) sec.null@ffffffffc0e99ea0: being destroyed 00000020:00000001:2.0:1773941619.994789:0:14694:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941619.994790:0:14694:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=1 obd=MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941619.994792:0:14694:0:(genops.c:1249:class_import_put()) Process leaving 10000000:00000001:2.0:1773941619.994794:0:14694:0:(obd_class.h:670:obd_cleanup_client_import()) Process leaving 10000000:00000001:2.0:1773941619.994796:0:14694:0:(mgc_request.c:774:mgc_llog_fini()) Process entered 10000000:00000040:2.0:1773941619.994799:0:14694:0:(lustre_log.h:385:llog_ctxt_get()) GETting ctxt ffff988d02eb6540 : new refcount 2 00000040:00000001:2.0:1773941619.994801:0:14694:0:(llog_obd.c:93:llog_cleanup()) Process entered 00000040:00000040:2.0:1773941619.994804:0:14694:0:(lustre_log.h:400:llog_ctxt_put()) PUTting ctxt ffff988d02eb6540 : new refcount 1 00000020:00000040:2.0:1773941619.994807:0:14694:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b5800 : new refcount 1 00000020:00000001:2.0:1773941619.994810:0:14694:0:(genops.c:1236:class_import_put()) Process entered 00000020:00000040:2.0:1773941619.994811:0:14694:0:(genops.c:1242:class_import_put()) import ffff988d20336000 refcount=0 obd=MGC192.168.204.114@tcp 00000020:00000040:2.0:1773941619.994814:0:14694:0:(genops.c:1245:class_import_put()) final put import ffff988d20336000 00000020:00000001:2.0:1773941619.994831:0:14694:0:(genops.c:1249:class_import_put()) Process leaving 00000040:00000010:2.0:1773941619.994834:0:14694:0:(llog_obd.c:46:llog_ctxt_destroy()) kfreed 'ctxt': 168 at ffff988d02eb6540. 00000040:00000001:2.0:1773941619.994847:0:14694:0:(llog_obd.c:122:llog_cleanup()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941619.994850:0:14694:0:(mgc_request.c:780:mgc_llog_fini()) Process leaving (rc=0 : 0 : 0) 10000000:00000001:2.0:1773941619.994853:0:14694:0:(mgc_request.c:816:mgc_device_fini()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1773941619.994859:0:14694:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff988d054378c0. 00000020:00000010:2.0:1773941619.994874:0:14694:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d03356400. 00000020:00000001:0.0:1773941619.994881:0:36:0:(genops.c:1200:obd_zombie_import_free()) Process entered 00000040:00000010:2.0:1773941619.994882:0:14694:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d03356600. 00000020:00000080:0.0:1773941619.994891:0:36:0:(genops.c:1202:obd_zombie_import_free()) destroying import ffff988d20336000 for MGC192.168.204.114@tcp 00000020:00000010:2.0:1773941619.994892:0:14694:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941619.994914:0:14694:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d03357e00. 00000020:00000040:0.0:1773941619.994915:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 1 to 192.168.204.114@tcp 00040000:00000010:2.0:1773941619.994948:0:14694:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d0a0db600. 00000020:00000040:0.0:1773941619.994958:0:36:0:(lustre_net.h:1981:ptlrpc_connection_put()) PUT conn=ffff988d08249140 refcount 0 to 192.168.204.114@tcp 00040000:00000010:2.0:1773941619.995015:0:14694:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d03356000. 00000020:00000010:2.0:1773941619.995025:0:14694:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d03357800. 00000020:00000001:2.0:1773941619.995030:0:14694:0:(obd_class.h:623:obd_precleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1773941619.995038:0:36:0:(genops.c:1213:obd_zombie_import_free()) kfreed 'imp_conn': 88 at ffff988d06580720. 00000020:00000040:0.0:1773941619.995048:0:36:0:(obd_config.c:1058:class_decref()) Decref MGC192.168.204.114@tcp (ffff988d2039cd28) now 4 - import 00000020:00000010:0.0:1773941619.995054:0:36:0:(genops.c:1220:obd_zombie_import_free()) kfreed 'imp': 1712 at ffff988d20336000. 00000020:00000001:0.0:1773941619.995065:0:36:0:(genops.c:1221:obd_zombie_import_free()) Process leaving 00000001:00000001:2.0:1773941619.995070:0:14694:0:(hash.c:1113:cfs_hash_destroy()) Process entered 00000001:00000010:2.0:1773941619.995080:0:14694:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff988d03357000. 00000001:00000010:2.0:1773941619.995085:0:14694:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff988d03357a00. 00000001:00000010:2.0:1773941619.995088:0:14694:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff988d03356c00. 00000001:00000010:2.0:1773941619.995091:0:14694:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 344 at ffff988d03356a00. 00000001:00000010:2.0:1773941619.995094:0:14694:0:(hash.c:913:cfs_hash_buckets_free()) kfreed 'buckets': 32 at ffff988d06545400. 00000001:00000010:2.0:1773941619.995097:0:14694:0:(hash.c:1163:cfs_hash_destroy()) kfreed 'hs': 232 at ffff988d04c1a300. 00000001:00000001:2.0:1773941619.995102:0:14694:0:(hash.c:1165:cfs_hash_destroy()) Process leaving 00000020:00000040:2.0:1773941619.995105:0:14694:0:(obd_config.c:1058:class_decref()) Decref MGC192.168.204.114@tcp (ffff988d2039cd28) now 3 - setup 00000020:00000001:2.0:1773941619.995108:0:14694:0:(obd_config.c:1021:class_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941619.995111:0:14694:0:(obd_config.c:1659:class_process_config()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000080:2.0:1773941619.995115:0:14694:0:(obd_config.c:1506:class_process_config()) processing cmd: cf002 00000020:00000001:2.0:1773941619.995119:0:14694:0:(obd_config.c:899:class_detach()) Process entered 00000020:00000040:2.0:1773941619.995123:0:14694:0:(obd_config.c:1058:class_decref()) Decref MGC192.168.204.114@tcp (ffff988d2039cd28) now 2 - obd_device_list 00000020:00000080:2.0:1773941619.995127:0:14694:0:(obd_config.c:919:class_detach()) detach on obd MGC192.168.204.114@tcp (uuid c1c1ff21-5f68-4bf6-bdc4-258644b24465) 00000020:00000040:2.0:1773941619.995129:0:14694:0:(obd_config.c:1058:class_decref()) Decref MGC192.168.204.114@tcp (ffff988d2039cd28) now 1 - newdev 00000020:00000040:2.0:1773941619.995133:0:14694:0:(lustre_handles.c:91:class_handle_unhash_nolock()) removing object ffff988d111b5800 with handle 0x8e86767622e28cd from hash 00000020:00000040:2.0:1773941619.995138:0:14694:0:(genops.c:898:class_export_put()) PUTting export ffff988d111b5800 : new refcount 0 00000020:00000080:2.0:1773941619.995140:0:14694:0:(genops.c:904:class_export_put()) final put ffff988d111b5800/c1c1ff21-5f68-4bf6-bdc4-258644b24465 00000020:00000001:2.0:1773941619.995142:0:14694:0:(lprocfs_status_server.c:686:lprocfs_exp_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1773941619.995146:0:14694:0:(genops.c:857:class_export_destroy()) Process entered 00000020:00000080:2.0:1773941619.995148:0:14694:0:(genops.c:863:class_export_destroy()) destroying export ffff988d111b5800/c1c1ff21-5f68-4bf6-bdc4-258644b24465 for MGC192.168.204.114@tcp 00000020:00000001:2.0:1773941619.995161:0:14694:0:(obd_class.h:1017:obd_destroy_export()) Process entered 00000020:00000001:2.0:1773941619.995164:0:14694:0:(obd_class.h:1021:obd_destroy_export()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:2.0:1773941619.995167:0:14694:0:(genops.c:879:class_export_destroy()) kfreed_rcu 'exp': 1208 at ffff988d111b5800. 00000020:00000001:2.0:1773941619.995173:0:14694:0:(genops.c:880:class_export_destroy()) Process leaving 00000020:00000040:2.0:1773941619.995176:0:14694:0:(genops.c:437:class_free_dev()) Release obd device MGC192.168.204.114@tcp obd_type name = mgc 00000020:01000000:2.0:1773941619.995180:0:14694:0:(genops.c:440:class_free_dev()) finishing cleanup of obd MGC192.168.204.114@tcp (c1c1ff21-5f68-4bf6-bdc4-258644b24465) 00000020:00000001:2.0:1773941619.995182:0:14694:0:(obd_class.h:633:obd_cleanup()) Process entered 00000020:00000010:2.0:1773941619.995186:0:14694:0:(lu_object.c:1762:keys_init()) kmalloced '(ctx->lc_value)': 320 at ffff988d03356a00. 00000020:00000010:2.0:1773941619.995190:0:14694:0:(lu_object.c:566:lu_global_key_init()) kmalloced '(value)': 512 at ffff988d03356c00. 00000040:00000010:2.0:1773941619.995193:0:14694:0:(llog_obd.c:216:llog_key_init()) kmalloced '(value)': 480 at ffff988d03357a00. 00000020:00000010:2.0:1773941619.995196:0:14694:0:(dt_object.c:32:dt_global_key_init()) kmalloced '(value)': 1264 at ffff988d09305000. 00000020:00000010:2.0:1773941619.995200:0:14694:0:(update_records.c:1195:update_key_init()) kmalloced '(value)': 432 at ffff988d03357000. 00040000:00000010:2.0:1773941619.995204:0:14694:0:(lquota_lib.c:35:lquota_key_init()) kmalloced '(value)': 248 at ffff988d0a0db600. 00040000:00000010:2.0:1773941619.995207:0:14694:0:(qsd_lib.c:51:qsd_key_init()) kmalloced '(value)': 408 at ffff988d03357800. 10000000:00000001:2.0:1773941619.995212:0:14694:0:(mgc_request.c:824:mgc_device_free()) Process entered 00000020:00000001:2.0:1773941619.995214:0:14694:0:(obd_config.c:1281:class_del_profiles()) Process entered 00000020:00000001:2.0:1773941619.995217:0:14694:0:(obd_config.c:1294:class_del_profiles()) Process leaving 00010000:00000001:2.0:1773941619.995354:0:14694:0:(ldlm_lib.c:631:client_obd_cleanup()) Process entered 00010000:00000001:2.0:1773941619.995359:0:14694:0:(ldlm_resource.c:1304:ldlm_namespace_free_post()) Process entered 00010000:00000001:2.0:1773941619.995367:0:14694:0:(ldlm_pool.c:931:ldlm_pool_fini()) Process entered 00000020:00000010:2.0:1773941619.995415:0:14694:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 448 at ffff988d065c4600. 00000020:00000010:2.0:1773941619.995425:0:14694:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 352 at ffff988d02cbde00. 00000020:00000010:2.0:1773941619.995432:0:14694:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff988d02ce5400. 00010000:00000001:2.0:1773941619.995479:0:14694:0:(ldlm_pool.c:934:ldlm_pool_fini()) Process leaving 00000020:00000010:2.0:1773941619.995498:0:14694:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 128 at ffff988d181fed80. 00000020:00000010:2.0:1773941619.995505:0:14694:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 128 at ffff988d0716bc00. 00000020:00000010:2.0:1773941619.995509:0:14694:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 128 at ffff988d107f4c00. 00000020:00000010:2.0:1773941619.995513:0:14694:0:(lprocfs_status.c:1542:stats_free()) kfreed 'stats->ls_percpu[i]': 128 at ffff988d03580580. 00000020:00000010:2.0:1773941619.995517:0:14694:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 96 at ffff988d065807e0. 00000020:00000010:2.0:1773941619.995521:0:14694:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff988d02ce4400. 00000001:00000001:2.0:1773941619.995575:0:14694:0:(hash.c:1113:cfs_hash_destroy()) Process entered 00000001:00000010:2.0:1773941619.995579:0:14694:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 304 at ffff988d10400a00. 00000001:00000010:2.0:1773941619.995583:0:14694:0:(hash.c:911:cfs_hash_buckets_free()) kfreed 'buckets[i]': 304 at ffff988d10401a00. 00000001:00000010:2.0:1773941619.995587:0:14694:0:(hash.c:913:cfs_hash_buckets_free()) kfreed 'buckets': 16 at ffff988d065cd9c0. 00000001:00000010:2.0:1773941619.995602:0:14694:0:(hash.c:1163:cfs_hash_destroy()) kfreed 'hs': 280 at ffff988d10401400. 00000001:00000001:2.0:1773941619.995606:0:14694:0:(hash.c:1165:cfs_hash_destroy()) Process leaving 00010000:00000010:2.0:1773941619.995609:0:14694:0:(ldlm_resource.c:1327:ldlm_namespace_free_post()) kfreed 'ns': 848 at ffff988d1125a400. 00010000:00000001:2.0:1773941619.995613:0:14694:0:(ldlm_lockd.c:3044:ldlm_put_ref()) Process entered 00010000:00000001:2.0:1773941619.995616:0:14694:0:(ldlm_lockd.c:3058:ldlm_put_ref()) Process leaving 00010000:00000001:2.0:1773941619.995618:0:14694:0:(ldlm_resource.c:1329:ldlm_namespace_free_post()) Process leaving 00010000:00000001:2.0:1773941619.995622:0:14694:0:(obd_class.h:652:obd_cleanup_client_import()) Process entered 00010000:00000001:2.0:1773941619.995624:0:14694:0:(obd_class.h:670:obd_cleanup_client_import()) Process leaving 00010000:00000001:2.0:1773941619.995626:0:14694:0:(ldlm_lockd.c:3044:ldlm_put_ref()) Process entered 00010000:00000001:2.0:1773941619.995629:0:14694:0:(ldlm_lockd.c:3527:ldlm_cleanup()) Process entered 00000001:00000010:2.0:1773941619.995646:0:14694:0:(shrinker_debug.c:208:ll_shrinker_free()) kfreed 's': 88 at ffff988d06580a80. 00000001:00000010:2.0:1773941619.995651:0:14694:0:(shrinker_debug.c:208:ll_shrinker_free()) kfreed 's': 88 at ffff988d06580840. 00000020:00000001:1.0:1773941619.995753:0:14706:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1773941619.995759:0:14706:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1773941619.995764:0:14706:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (-1 blwis in pool) 00010000:00000001:1.0:1773941619.995769:0:14706:0:(ldlm_lockd.c:2874:ldlm_bl_thread_blwi()) Process entered 00010000:00000001:1.0:1773941619.995772:0:14706:0:(ldlm_lockd.c:2878:ldlm_bl_thread_blwi()) Process leaving (rc=2 : 2 : 2) 00000020:00000010:1.0:1773941619.995789:0:14706:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff988d03ec79c0. 00000020:00000010:1.0:1773941619.995803:0:14706:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d02d19400. 00000020:00000010:1.0:1773941619.995810:0:14706:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d08359000. 00000020:00000010:1.0:1773941619.995817:0:14706:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff988d03342840. 00000020:00000010:1.0:1773941619.995823:0:14706:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d02d18400. 80000000:00000010:1.0:1773941619.995828:0:14706:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d111f7ea0. 40000000:00000010:1.0:1773941619.995834:0:14706:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff988d03ec7800. 00100000:00000010:1.0:1773941619.995840:0:14706:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff988d0835c000. 00040000:00000010:1.0:1773941619.995846:0:14706:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d21731700. 00040000:00000010:1.0:1773941619.995851:0:14706:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d02d18c00. 00002000:00000010:1.0:1773941619.995858:0:14706:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff988d03521800. 00000020:00000010:1.0:1773941619.995862:0:14706:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d02d19600. 00010000:00000010:1.0:1773941619.995867:0:14706:0:(ldlm_lockd.c:3018:ldlm_bl_thread_main()) kfreed 'env': 56 at ffff988d03ec7d40. 00010000:00000001:1.0:1773941619.995871:0:14706:0:(ldlm_lockd.c:3019:ldlm_bl_thread_main()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1773941619.995901:0:14705:0:(genops.c:1914:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1773941619.995907:0:14705:0:(genops.c:1928:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1773941619.995913:0:14705:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 0 locks of 0 total in blp. (-2 blwis in pool) 00010000:00000001:3.0:1773941619.995942:0:14705:0:(ldlm_lockd.c:2874:ldlm_bl_thread_blwi()) Process entered 00010000:00000001:3.0:1773941619.995945:0:14705:0:(ldlm_lockd.c:2878:ldlm_bl_thread_blwi()) Process leaving (rc=2 : 2 : 2) 00000020:00000010:3.0:1773941619.995992:0:14705:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff988d03ec75c0. 00000020:00000010:3.0:1773941619.996023:0:14705:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d1817d600. 00000020:00000010:3.0:1773941619.996036:0:14705:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d0835b800. 00010000:00000010:2.0:1773941619.996040:0:14694:0:(ldlm_lockd.c:3556:ldlm_cleanup()) kfreed 'blp': 168 at ffff988d02eb6180. 00000020:00000010:3.0:1773941619.996046:0:14705:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff988d03340f18. 00000100:00000001:2.0:1773941619.996051:0:14694:0:(service.c:3841:ptlrpc_unregister_service()) Process entered 00000100:00000200:2.0:1773941619.996057:0:14694:0:(service.c:3843:ptlrpc_unregister_service()) ldlm_cbd: tearing down 00000020:00000010:3.0:1773941619.996059:0:14705:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d04f8ae00. 00000100:00000001:2.0:1773941619.996066:0:14694:0:(service.c:3384:ptlrpc_stop_all_threads()) Process entered 80000000:00000010:3.0:1773941619.996067:0:14705:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d111f7000. 00000100:00000001:2.0:1773941619.996073:0:14694:0:(service.c:3338:ptlrpc_svcpt_stop_threads()) Process entered 40000000:00000010:3.0:1773941619.996076:0:14705:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff988d03ec7080. 00000100:00000040:2.0:1773941619.996079:0:14694:0:(service.c:3341:ptlrpc_svcpt_stop_threads()) Stopping threads for service ldlm_cbd 00100000:00000010:3.0:1773941619.996084:0:14705:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff988d0835a000. 00000100:00000040:2.0:1773941619.996086:0:14694:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cb #1 00000100:00000040:2.0:1773941619.996093:0:14694:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cb #0 00040000:00000010:3.0:1773941619.996094:0:14705:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d21731c00. 00040000:00000010:3.0:1773941619.996101:0:14705:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d02d18800. 00002000:00000010:3.0:1773941619.996105:0:14705:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff988d03521000. 00000100:00000040:2.0:1773941619.996118:0:14694:0:(service.c:3360:ptlrpc_svcpt_stop_threads()) waiting for stopping-thread ldlm_cb #1 00000020:00000010:3.0:1773941619.996123:0:14705:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d1a204a00. 00010000:00000010:3.0:1773941619.996130:0:14705:0:(ldlm_lockd.c:3018:ldlm_bl_thread_main()) kfreed 'env': 56 at ffff988d03ec7600. 00010000:00000001:3.0:1773941619.996137:0:14705:0:(ldlm_lockd.c:3019:ldlm_bl_thread_main()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1773941619.996191:0:14695:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d11267a00. 00000020:00000010:1.0:1773941619.996192:0:14696:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d0a06d200. 00000040:00000010:1.0:1773941619.996230:0:14696:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d0a06d600. 00000040:00000010:0.0:1773941619.996230:0:14695:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d11266a00. 00000020:00000010:0.0:1773941619.996264:0:14695:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d111b2800. 00000020:00000010:1.0:1773941619.996265:0:14696:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d08af4800. 00000020:00000010:1.0:1773941619.996291:0:14696:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff988d054edf98. 00000020:00000010:0.0:1773941619.996292:0:14695:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff988d054eeeb0. 00000020:00000010:1.0:1773941619.996320:0:14696:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d0a06d800. 00000020:00000010:0.0:1773941619.996321:0:14695:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d0831ba00. 80000000:00000010:0.0:1773941619.996337:0:14695:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d06580060. 80000000:00000010:1.0:1773941619.996342:0:14696:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d06580660. 40000000:00000010:0.0:1773941619.996347:0:14695:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff988d10872140. 40000000:00000010:1.0:1773941619.996365:0:14696:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff988d108721c0. 00100000:00000010:0.0:1773941619.996365:0:14695:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff988d111b5000. 00040000:00000010:0.0:1773941619.996383:0:14695:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d04c1a100. 00100000:00000010:1.0:1773941619.996389:0:14696:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff988d08af3000. 00040000:00000010:0.0:1773941619.996395:0:14695:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff988d111b1000. 00040000:00000010:1.0:1773941619.996399:0:14696:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d04c1a600. 00040000:00000010:0.0:1773941619.996405:0:14695:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d02c84c00. 00040000:00000010:1.0:1773941619.996411:0:14696:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff988d08af6000. 00000004:00000010:0.0:1773941619.996415:0:14695:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff988d111b6000. 00040000:00000010:1.0:1773941619.996420:0:14696:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d063a9400. 00000004:00000010:0.0:1773941619.996426:0:14695:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff988d18a23000. 00000004:00000010:0.0:1773941619.996434:0:14695:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff988d111b3800. 00002000:00000010:0.0:1773941619.996447:0:14695:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff988d1081d400. 00000004:00000010:1.0:1773941619.996452:0:14696:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff988d08af4000. 00000004:00000010:0.0:1773941619.996455:0:14695:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff988d1081e000. 00000004:00000010:1.0:1773941619.996470:0:14696:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff988d034b4000. 00000020:00000010:0.0:1773941619.996472:0:14695:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d11267c00. 00000004:00000010:1.0:1773941619.996507:0:14696:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff988d08af7000. 00000020:00000010:0.0:1773941619.996508:0:14695:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff988d10872700. 00002000:00000010:1.0:1773941619.996525:0:14696:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff988d1081dc00. 00000100:00000010:0.0:1773941619.996529:0:14695:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff988d10872080. 00000004:00000010:1.0:1773941619.996534:0:14696:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff988d1081e400. 00000100:00100000:0.0:1773941619.996538:0:14695:0:(service.c:3162:ptlrpc_main()) ldlm_cb00_000: service thread [ffff988d11267600:14695] 0 exiting: rc = 1 00000020:00000010:1.0:1773941619.996543:0:14696:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d02c84600. 00000020:00000010:1.0:1773941619.996553:0:14696:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff988d10872300. 00000100:00000010:1.0:1773941619.996567:0:14696:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff988d108722c0. 00000100:00100000:1.0:1773941619.996576:0:14696:0:(service.c:3162:ptlrpc_main()) ldlm_cb00_001: service thread [ffff988d1817c200:14696] 1 exiting: rc = 0 00000100:00000010:2.0:1773941619.996694:0:14694:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff988d11267600. 00000100:00000010:2.0:1773941619.996706:0:14694:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff988d1817c200. 00000100:00000001:2.0:1773941619.996710:0:14694:0:(service.c:3375:ptlrpc_svcpt_stop_threads()) Process leaving 00000100:00000001:2.0:1773941619.996713:0:14694:0:(service.c:3338:ptlrpc_svcpt_stop_threads()) Process entered 00000100:00000040:2.0:1773941619.996715:0:14694:0:(service.c:3341:ptlrpc_svcpt_stop_threads()) Stopping threads for service ldlm_cbd 00000100:00000040:2.0:1773941619.996717:0:14694:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cb #1 00000100:00000040:2.0:1773941619.996720:0:14694:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cb #0 00000100:00000040:2.0:1773941619.996744:0:14694:0:(service.c:3360:ptlrpc_svcpt_stop_threads()) waiting for stopping-thread ldlm_cb #1 00000020:00000010:3.0:1773941619.996791:0:14697:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d1817c600. 00000040:00000010:3.0:1773941619.996802:0:14697:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d1817c000. 00000020:00000010:3.0:1773941619.996806:0:14697:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d07233000. 00000020:00000010:3.0:1773941619.996813:0:14697:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff988d03341e30. 00000020:00000010:3.0:1773941619.996819:0:14697:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d1817cc00. 80000000:00000010:3.0:1773941619.996823:0:14697:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d111f7ae0. 40000000:00000010:3.0:1773941619.996827:0:14697:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff988d03ec7c00. 00100000:00000010:3.0:1773941619.996831:0:14697:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff988d07230800. 00040000:00000010:3.0:1773941619.996835:0:14697:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d013d4000. 00040000:00000010:3.0:1773941619.996839:0:14697:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff988d07234800. 00040000:00000010:3.0:1773941619.996843:0:14697:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d1817ce00. 00000004:00000010:3.0:1773941619.996847:0:14697:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff988d07236800. 00000004:00000010:3.0:1773941619.996851:0:14697:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff988d02d6f000. 00000004:00000010:3.0:1773941619.996855:0:14697:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff988d07237000. 00002000:00000010:3.0:1773941619.996859:0:14697:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff988d03522000. 00000004:00000010:3.0:1773941619.996863:0:14697:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff988d03521400. 00000020:00000010:3.0:1773941619.996866:0:14697:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d1817c400. 00000020:00000010:3.0:1773941619.996871:0:14697:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff988d03ec7840. 00000100:00000010:3.0:1773941619.996878:0:14697:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff988d03ec7a80. 00000100:00100000:3.0:1773941619.996881:0:14697:0:(service.c:3162:ptlrpc_main()) ldlm_cb01_000: service thread [ffff988d1817d800:14697] 0 exiting: rc = 1 00000020:00000010:3.0:1773941619.997027:0:14698:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d1817c800. 00000040:00000010:3.0:1773941619.997036:0:14698:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d1817d200. 00000020:00000010:3.0:1773941619.997043:0:14698:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d0835d800. 00000020:00000010:3.0:1773941619.997049:0:14698:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff988d033469a8. 00000020:00000010:3.0:1773941619.997056:0:14698:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d1817ca00. 80000000:00000010:3.0:1773941619.997063:0:14698:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d111f7300. 40000000:00000010:3.0:1773941619.997068:0:14698:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff988d03ec7cc0. 00100000:00000010:3.0:1773941619.997073:0:14698:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff988d0835e800. 00040000:00000010:3.0:1773941619.997080:0:14698:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d21731a00. 00040000:00000010:3.0:1773941619.997086:0:14698:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff988d08359800. 00040000:00000010:3.0:1773941619.997092:0:14698:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d1817d000. 00000004:00000010:3.0:1773941619.997114:0:14698:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff988d08358800. 00000004:00000010:3.0:1773941619.997120:0:14698:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff988d02d6c000. 00000004:00000010:3.0:1773941619.997126:0:14698:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff988d08358000. 00002000:00000010:3.0:1773941619.997133:0:14698:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff988d03520800. 00000004:00000010:3.0:1773941619.997139:0:14698:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff988d03523c00. 00000020:00000010:3.0:1773941619.997145:0:14698:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d1817dc00. 00000020:00000010:3.0:1773941619.997154:0:14698:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff988d03ec7180. 00000100:00000010:3.0:1773941619.997166:0:14698:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff988d03ec7900. 00000100:00100000:3.0:1773941619.997172:0:14698:0:(service.c:3162:ptlrpc_main()) ldlm_cb01_001: service thread [ffff988d063a9000:14698] 1 exiting: rc = 0 00000100:00000010:2.0:1773941619.997266:0:14694:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff988d1817d800. 00000100:00000010:2.0:1773941619.997273:0:14694:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff988d063a9000. 00000100:00000001:2.0:1773941619.997282:0:14694:0:(service.c:3375:ptlrpc_svcpt_stop_threads()) Process leaving 00000100:00000001:2.0:1773941619.997284:0:14694:0:(service.c:3391:ptlrpc_stop_all_threads()) Process leaving 00000400:00000200:2.0:1773941619.997290:0:14694:0:(lib-ptl.c:969:lnet_clear_lazy_portal()) clearing portal 15 lazy 00000400:00000001:2.0:1773941619.997295:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941619.997297:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941619.997300:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037fbb0. 00000400:00000200:2.0:1773941619.997317:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04facc00 00000400:00000010:2.0:1773941619.997321:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d04facc00. 00000100:00000001:2.0:1773941619.997328:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1773941619.997332:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_cbd 00000100:00100000:2.0:1773941619.997345:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:2.0:1773941619.997350:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 0 buffers still posted 00000100:00000001:2.0:1773941619.997355:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:2.0:1773941619.997360:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1773941619.997362:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:2.0:1773941619.997366:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d03ccddc0. 00000400:00000200:2.0:1773941619.997371:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1837de40 00000400:00000010:2.0:1773941619.997375:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1837de40. 00000100:00000001:2.0:1773941619.997380:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:2.0:1773941619.997383:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_cbd 00000100:00100000:2.0:1773941619.997392:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:2.0:1773941619.997395:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 0 buffers still posted 00000100:00000001:2.0:1773941619.997398:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:2.0:1773941619.997404:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1773941619.997408:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000010:2.0:1773941619.997419:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d10806000. 00000100:00000010:2.0:1773941619.997427:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d065f9c00. 00000100:00000010:2.0:1773941619.997432:0:14694:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff988d08af3800. 00000100:00000010:2.0:1773941619.997436:0:14694:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff988d180dc800. 00000100:00000001:2.0:1773941619.997440:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:2.0:1773941619.997441:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000010:2.0:1773941619.997445:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d10802000. 00000100:00000010:2.0:1773941619.997448:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d065f8000. 00000100:00000010:2.0:1773941619.997452:0:14694:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff988d0835d000. 00000100:00000010:2.0:1773941619.997455:0:14694:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff988d0835a800. 00000100:00000001:2.0:1773941619.997459:0:14694:0:(nrs.c:1051:nrs_svcpt_cleanup_locked()) Process entered 00000100:00000001:2.0:1773941619.997462:0:14694:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:2.0:1773941619.997467:0:14694:0:(nrs.c:111:nrs_policy_stop_locked()) Process entered 00000100:00000001:2.0:1773941619.997470:0:14694:0:(nrs.c:67:nrs_policy_stop0()) Process entered 00000100:00000010:2.0:1773941619.997474:0:14694:0:(nrs_fifo.c:83:nrs_fifo_stop()) kfreed 'head': 40 at ffff988d10872f00. 00000100:00000001:2.0:1773941619.997478:0:14694:0:(nrs.c:85:nrs_policy_stop0()) Process leaving 00000100:00000001:2.0:1773941619.997480:0:14694:0:(nrs.c:147:nrs_policy_stop_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941619.997483:0:14694:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:2.0:1773941619.997496:0:14694:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d04fa29c0. 00000100:00000001:2.0:1773941619.997499:0:14694:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:2.0:1773941619.997501:0:14694:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:2.0:1773941619.997503:0:14694:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d04fa2480. 00000100:00000001:2.0:1773941619.997506:0:14694:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:2.0:1773941619.997507:0:14694:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:2.0:1773941619.997509:0:14694:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d04fa2180. 00000100:00000001:2.0:1773941619.997511:0:14694:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:2.0:1773941619.997513:0:14694:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:2.0:1773941619.997514:0:14694:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d04fa2240. 00000100:00000001:2.0:1773941619.997517:0:14694:0:(nrs.c:1082:nrs_svcpt_cleanup_locked()) Process leaving 00000100:00000001:2.0:1773941619.997519:0:14694:0:(nrs.c:1051:nrs_svcpt_cleanup_locked()) Process entered 00000100:00000001:2.0:1773941619.997520:0:14694:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:2.0:1773941619.997521:0:14694:0:(nrs.c:111:nrs_policy_stop_locked()) Process entered 00000100:00000001:2.0:1773941619.997523:0:14694:0:(nrs.c:67:nrs_policy_stop0()) Process entered 00000100:00000010:2.0:1773941619.997525:0:14694:0:(nrs_fifo.c:83:nrs_fifo_stop()) kfreed 'head': 40 at ffff988d10872a40. 00000100:00000001:2.0:1773941619.997527:0:14694:0:(nrs.c:85:nrs_policy_stop0()) Process leaving 00000100:00000001:2.0:1773941619.997529:0:14694:0:(nrs.c:147:nrs_policy_stop_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941619.997531:0:14694:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:2.0:1773941619.997532:0:14694:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d04fac180. 00000100:00000001:2.0:1773941619.997535:0:14694:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:2.0:1773941619.997536:0:14694:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:2.0:1773941619.997538:0:14694:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d04fac780. 00000100:00000001:2.0:1773941619.997540:0:14694:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:2.0:1773941619.997542:0:14694:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:2.0:1773941619.997544:0:14694:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d04facb40. 00000100:00000001:2.0:1773941619.997546:0:14694:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:2.0:1773941619.997548:0:14694:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:2.0:1773941619.997550:0:14694:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d04fac6c0. 00000100:00000001:2.0:1773941619.997552:0:14694:0:(nrs.c:1082:nrs_svcpt_cleanup_locked()) Process leaving 00000020:00000010:2.0:1773941619.997668:0:14694:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 3680 at ffff988d18a21000. 00000020:00000010:2.0:1773941619.997679:0:14694:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff988d065f8c00. 00000100:00000010:2.0:1773941619.997723:0:14694:0:(service.c:3818:ptlrpc_service_free()) kfreed 'array->paa_reqs_array': 12080 at ffff988d0653c000. 00000100:00000010:2.0:1773941619.997745:0:14694:0:(service.c:3824:ptlrpc_service_free()) kfreed 'array->paa_reqs_count': 3020 at ffff988d18a24000. 00000100:00000010:2.0:1773941619.997753:0:14694:0:(service.c:3818:ptlrpc_service_free()) kfreed 'array->paa_reqs_array': 12080 at ffff988d18b48000. 00000100:00000010:2.0:1773941619.997770:0:14694:0:(service.c:3824:ptlrpc_service_free()) kfreed 'array->paa_reqs_count': 3020 at ffff988d18a27000. 00000100:00000010:2.0:1773941619.997776:0:14694:0:(service.c:3830:ptlrpc_service_free()) kfreed 'svcpt': 784 at ffff988d065fa000. 00000100:00000010:2.0:1773941619.997782:0:14694:0:(service.c:3830:ptlrpc_service_free()) kfreed 'svcpt': 784 at ffff988d065fb000. 00000100:00000010:2.0:1773941619.997799:0:14694:0:(service.c:3836:ptlrpc_service_free()) kfreed 'svc': 360 at ffff988d11267000. 00000100:00000001:2.0:1773941619.997805:0:14694:0:(service.c:3863:ptlrpc_unregister_service()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1773941619.997809:0:14694:0:(service.c:3841:ptlrpc_unregister_service()) Process entered 00000100:00000200:2.0:1773941619.997812:0:14694:0:(service.c:3843:ptlrpc_unregister_service()) ldlm_canceld: tearing down 00000100:00000001:2.0:1773941619.997819:0:14694:0:(service.c:3384:ptlrpc_stop_all_threads()) Process entered 00000100:00000001:2.0:1773941619.997822:0:14694:0:(service.c:3338:ptlrpc_svcpt_stop_threads()) Process entered 00000100:00000040:2.0:1773941619.997825:0:14694:0:(service.c:3341:ptlrpc_svcpt_stop_threads()) Stopping threads for service ldlm_canceld 00000100:00000040:2.0:1773941619.997829:0:14694:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cn #2 00000100:00000040:2.0:1773941619.997832:0:14694:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cn #1 00000100:00000040:2.0:1773941619.997835:0:14694:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cn #0 00000100:00000040:2.0:1773941619.997864:0:14694:0:(service.c:3360:ptlrpc_svcpt_stop_threads()) waiting for stopping-thread ldlm_cn #2 00000020:00000010:1.0:1773941619.997911:0:14700:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d08a4f800. 00000020:00000010:1.0:1773941619.997942:0:14700:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d054f3c00. 00000040:00000010:1.0:1773941619.997955:0:14700:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d08a4e800. 00000020:00000010:0.0:1773941619.997956:0:14699:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d1961fa00. 00000020:00000010:1.0:1773941619.997998:0:14700:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d18b1e800. 00000020:00000010:1.0:1773941619.998006:0:14700:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff988d181f3250. 00000020:00000010:1.0:1773941619.998013:0:14700:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d08a4e400. 00010000:00000010:0.1:1773941619.998018:0:14699:0:(ldlm_resource.c:1519:__ldlm_resource_free()) slab-freed '(res)': 248 at ffff988d21ec1400. 80000000:00000010:1.0:1773941619.998021:0:14700:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d054f3000. 40000000:00000010:1.0:1773941619.998032:0:14700:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff988d04d67480. 00010000:00000010:0.1:1773941619.998036:0:14699:0:(ldlm_lock.c:174:lock_handle_free()) slab-freed 'lock': 512 at ffff988d00031400. 00008000:00000010:1.0:1773941619.998040:0:14700:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d0833b880. 00100000:00000010:1.0:1773941619.998048:0:14700:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff988d18b1d800. 00040000:00000010:1.0:1773941619.998059:0:14700:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d02cfd000. 00000020:00000010:0.0:1773941619.998063:0:14699:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d06580600. 00040000:00000010:1.0:1773941619.998066:0:14700:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff988d18b1c800. 00040000:00000010:1.0:1773941619.998071:0:14700:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d08a4e200. 00000004:00000010:1.0:1773941619.998074:0:14700:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff988d18b1b000. 00000004:00000010:1.0:1773941619.998078:0:14700:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff988d065f6000. 00000004:00000010:1.0:1773941619.998094:0:14700:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff988d18b1c000. 00000040:00000010:0.0:1773941619.998094:0:14699:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d1961ea00. 00002000:00000010:1.0:1773941619.998123:0:14700:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff988d11189800. 00000020:00000010:0.0:1773941619.998125:0:14699:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d08af5800. 00000004:00000010:1.0:1773941619.998154:0:14700:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff988d1118ac00. 00000020:00000010:0.0:1773941619.998155:0:14699:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff988d054ead48. 00000020:00000010:1.0:1773941619.998179:0:14700:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d08a4f000. 00000020:00000010:0.0:1773941619.998180:0:14699:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d217b6600. 80000000:00000010:0.0:1773941619.998208:0:14699:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d065801e0. 00000020:00000010:1.0:1773941619.998212:0:14700:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff988d04d67b40. 40000000:00000010:0.0:1773941619.998218:0:14699:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff988d10872780. 00000100:00000010:1.0:1773941619.998225:0:14700:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff988d04d67540. 00008000:00000010:0.0:1773941619.998229:0:14699:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d03335c40. 00000100:00100000:1.0:1773941619.998233:0:14700:0:(service.c:3162:ptlrpc_main()) ldlm_cn00_001: service thread [ffff988d08a4fc00:14700] 1 exiting: rc = 0 00100000:00000010:0.0:1773941619.998239:0:14699:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff988d20330000. 00040000:00000010:0.0:1773941619.998248:0:14699:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d02d87000. 00040000:00000010:0.0:1773941619.998260:0:14699:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff988d20331000. 00040000:00000010:0.0:1773941619.998266:0:14699:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d10401000. 00000004:00000010:0.0:1773941619.998273:0:14699:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff988d20332800. 00000004:00000010:0.0:1773941619.998279:0:14699:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff988d07205000. 00000004:00000010:0.0:1773941619.998287:0:14699:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff988d20335800. 00002000:00000010:0.0:1773941619.998296:0:14699:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff988d1125b800. 00000004:00000010:0.0:1773941619.998301:0:14699:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff988d1125b000. 00000020:00000010:0.0:1773941619.998307:0:14699:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d1961e200. 00000020:00000010:0.0:1773941619.998315:0:14699:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff988d10872cc0. 00000100:00000010:0.0:1773941619.998324:0:14699:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff988d10872d80. 00000100:00100000:0.0:1773941619.998330:0:14699:0:(service.c:3162:ptlrpc_main()) ldlm_cn00_000: service thread [ffff988d063a8000:14699] 0 exiting: rc = 1 00000020:00000010:1.0:1773941619.998370:0:14701:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d10401800. 00000020:00000010:1.0:1773941619.998385:0:14701:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d06580960. 00000040:00000010:1.0:1773941619.998396:0:14701:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d10400800. 00000020:00000010:1.0:1773941619.998420:0:14701:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d20330800. 00000020:00000010:1.0:1773941619.998430:0:14701:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff988d054ef3b8. 00000020:00000010:1.0:1773941619.998437:0:14701:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d10400400. 80000000:00000010:1.0:1773941619.998443:0:14701:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d06580f60. 40000000:00000010:1.0:1773941619.998446:0:14701:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff988d108725c0. 00008000:00000010:1.0:1773941619.998452:0:14701:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d03334710. 00100000:00000010:1.0:1773941619.998457:0:14701:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff988d20332000. 00040000:00000010:1.0:1773941619.998462:0:14701:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d02d87300. 00040000:00000010:1.0:1773941619.998466:0:14701:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff988d20336800. 00040000:00000010:1.0:1773941619.998470:0:14701:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d10401e00. 00000004:00000010:1.0:1773941619.998474:0:14701:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff988d20335000. 00000004:00000010:1.0:1773941619.998478:0:14701:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff988d02bc7000. 00000004:00000010:1.0:1773941619.998483:0:14701:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff988d20331800. 00002000:00000010:1.0:1773941619.998487:0:14701:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff988d11258000. 00000004:00000010:1.0:1773941619.998491:0:14701:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff988d1125a800. 00000020:00000010:1.0:1773941619.998494:0:14701:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d10400c00. 00000020:00000010:1.0:1773941619.998500:0:14701:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff988d10872280. 00000100:00000010:1.0:1773941619.998506:0:14701:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff988d108724c0. 00000100:00100000:1.0:1773941619.998510:0:14701:0:(service.c:3162:ptlrpc_main()) ldlm_cn00_002: service thread [ffff988d10401c00:14701] 2 exiting: rc = 0 00000100:00000010:2.0:1773941619.998561:0:14694:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff988d063a8000. 00000100:00000010:2.0:1773941619.998574:0:14694:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff988d08a4fc00. 00000100:00000010:2.0:1773941619.998581:0:14694:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff988d10401c00. 00000100:00000001:2.0:1773941619.998585:0:14694:0:(service.c:3375:ptlrpc_svcpt_stop_threads()) Process leaving 00000100:00000001:2.0:1773941619.998587:0:14694:0:(service.c:3338:ptlrpc_svcpt_stop_threads()) Process entered 00000100:00000040:2.0:1773941619.998589:0:14694:0:(service.c:3341:ptlrpc_svcpt_stop_threads()) Stopping threads for service ldlm_canceld 00000100:00000040:2.0:1773941619.998593:0:14694:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cn #2 00000100:00000040:2.0:1773941619.998596:0:14694:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cn #1 00000100:00000040:2.0:1773941619.998598:0:14694:0:(service.c:2800:ptlrpc_stop_thread()) Stopping thread ldlm_cn #0 00000100:00000040:2.0:1773941619.998626:0:14694:0:(service.c:3360:ptlrpc_svcpt_stop_threads()) waiting for stopping-thread ldlm_cn #2 00000020:00000010:2.0:1773941619.998641:0:14703:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d0350f000. 00000020:00000010:2.0:1773941619.998647:0:14703:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d08bbbb40. 00000040:00000010:2.0:1773941619.998652:0:14703:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d0350f800. 00000020:00000010:2.0:1773941619.998655:0:14703:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d0544c000. 00000020:00000010:2.0:1773941619.998661:0:14703:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff988d05400000. 00000020:00000010:2.0:1773941619.998666:0:14703:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d0350f600. 80000000:00000010:2.0:1773941619.998672:0:14703:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d08bbb6c0. 40000000:00000010:2.0:1773941619.998677:0:14703:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff988d05437e00. 00000020:00000010:3.0:1773941619.998681:0:14702:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d0350f400. 00008000:00000010:2.0:1773941619.998683:0:14703:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d08326350. 00000020:00000010:3.0:1773941619.998698:0:14702:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d08bbb960. 00100000:00000010:2.0:1773941619.998699:0:14703:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff988d0544b000. 00000040:00000010:3.0:1773941619.998712:0:14702:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d0350e600. 00040000:00000010:2.0:1773941619.998714:0:14703:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d0a0db100. 00000020:00000010:3.0:1773941619.998724:0:14702:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d05449800. 00040000:00000010:2.0:1773941619.998726:0:14703:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff988d0544c800. 00000020:00000010:3.0:1773941619.998737:0:14702:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff988d05400a10. 00040000:00000010:2.0:1773941619.998738:0:14703:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d0350e200. 00000020:00000010:3.0:1773941619.998753:0:14702:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d0350fa00. 00000004:00000010:2.0:1773941619.998753:0:14703:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff988d0544d800. 80000000:00000010:3.0:1773941619.998762:0:14702:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d08bbbd20. 40000000:00000010:3.0:1773941619.998773:0:14702:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff988d05437300. 00000004:00000010:2.0:1773941619.998778:0:14703:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff988d08afe000. 00008000:00000010:3.0:1773941619.998783:0:14702:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d08324710. 00000004:00000010:2.0:1773941619.998787:0:14703:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff988d0544e800. 00100000:00000010:3.0:1773941619.998792:0:14702:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff988d05449000. 00002000:00000010:2.0:1773941619.998796:0:14703:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff988d035a5400. 00040000:00000010:3.0:1773941619.998802:0:14702:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d217a8d00. 00000004:00000010:2.0:1773941619.998804:0:14703:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff988d035a6000. 00040000:00000010:3.0:1773941619.998815:0:14702:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff988d0544b800. 00000020:00000010:2.0:1773941619.998816:0:14703:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0350ec00. 00040000:00000010:3.0:1773941619.998827:0:14702:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d0350e000. 00000020:00000010:2.0:1773941619.998831:0:14703:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff988d054375c0. 00000004:00000010:3.0:1773941619.998836:0:14702:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff988d0544e000. 00000100:00000010:2.0:1773941619.998842:0:14703:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff988d05437640. 00000004:00000010:3.0:1773941619.998845:0:14702:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff988d08afb000. 00000100:00100000:2.0:1773941619.998850:0:14703:0:(service.c:3162:ptlrpc_main()) ldlm_cn01_001: service thread [ffff988d10400200:14703] 1 exiting: rc = 0 00000004:00000010:3.0:1773941619.998855:0:14702:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff988d0544f800. 00002000:00000010:3.0:1773941619.998864:0:14702:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff988d035a6800. 00000004:00000010:3.0:1773941619.998869:0:14702:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff988d035a5800. 00000020:00000010:3.0:1773941619.998876:0:14702:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d0350e800. 00000020:00000010:3.0:1773941619.998882:0:14702:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff988d05437040. 00000100:00000010:3.0:1773941619.998891:0:14702:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff988d05437340. 00000100:00100000:3.0:1773941619.998899:0:14702:0:(service.c:3162:ptlrpc_main()) ldlm_cn01_000: service thread [ffff988d08a4ee00:14702] 0 exiting: rc = 1 00000020:00000010:2.0:1773941619.998997:0:14704:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d0381f000. 00000020:00000010:2.0:1773941619.999009:0:14704:0:(cl_object.c:1137:cl_key_fini()) kfreed 'info': 72 at ffff988d08bbb5a0. 00000040:00000010:2.0:1773941619.999017:0:14704:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d0381ec00. 00000020:00000010:2.0:1773941619.999026:0:14704:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d05448000. 00000020:00000010:2.0:1773941619.999033:0:14704:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff988d05403c60. 00000020:00000010:2.0:1773941619.999044:0:14704:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d0381fe00. 80000000:00000010:2.0:1773941619.999054:0:14704:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d08bbb3c0. 40000000:00000010:2.0:1773941619.999058:0:14704:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff988d05437f40. 00008000:00000010:2.0:1773941619.999063:0:14704:0:(echo_client.c:424:echo_thread_key_fini()) slab-freed '(info)': 904 at ffff988d083251a8. 00100000:00000010:2.0:1773941619.999068:0:14704:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff988d0544a800. 00040000:00000010:2.0:1773941619.999074:0:14704:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d0a0db200. 00040000:00000010:2.0:1773941619.999078:0:14704:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff988d05448800. 00040000:00000010:2.0:1773941619.999082:0:14704:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d0381fa00. 00000004:00000010:2.0:1773941619.999089:0:14704:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff988d0544a000. 00000004:00000010:2.0:1773941619.999095:0:14704:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff988d08af8000. 00000004:00000010:2.0:1773941619.999099:0:14704:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff988d09304800. 00002000:00000010:2.0:1773941619.999105:0:14704:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff988d035a6c00. 00000004:00000010:2.0:1773941619.999110:0:14704:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff988d035a7000. 00000020:00000010:2.0:1773941619.999115:0:14704:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d07ebbc00. 00000020:00000010:2.0:1773941619.999124:0:14704:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff988d05437a40. 00000100:00000010:2.0:1773941619.999130:0:14704:0:(service.c:3155:ptlrpc_main()) kfreed 'env': 56 at ffff988d05437f80. 00000100:00100000:2.0:1773941619.999134:0:14704:0:(service.c:3162:ptlrpc_main()) ldlm_cn01_002: service thread [ffff988d08a4e000:14704] 2 exiting: rc = 0 00000100:00000010:3.0:1773941619.999194:0:14694:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff988d08a4ee00. 00000100:00000010:3.0:1773941619.999204:0:14694:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff988d10400200. 00000100:00000010:3.0:1773941619.999208:0:14694:0:(service.c:3373:ptlrpc_svcpt_stop_threads()) kfreed 'thread': 328 at ffff988d08a4e000. 00000100:00000001:3.0:1773941619.999223:0:14694:0:(service.c:3375:ptlrpc_svcpt_stop_threads()) Process leaving 00000100:00000001:3.0:1773941619.999225:0:14694:0:(service.c:3391:ptlrpc_stop_all_threads()) Process leaving 00000400:00000200:3.0:1773941619.999230:0:14694:0:(lib-ptl.c:969:lnet_clear_lazy_portal()) clearing portal 17 lazy 00000400:00000001:3.0:1773941619.999235:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941619.999236:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941619.999240:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0014b108. 00000400:00000200:3.0:1773941619.999243:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d2027ea80 00000400:00000010:3.0:1773941619.999246:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d2027ea80. 00000100:00000001:3.0:1773941619.999250:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941619.999253:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941619.999261:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941619.999263:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:3.0:1773941619.999266:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941619.999268:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941619.999270:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941619.999272:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d1121ea50. 00000400:00000200:3.0:1773941619.999275:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d2027ecc0 00000400:00000010:3.0:1773941619.999277:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d2027ecc0. 00000100:00000001:3.0:1773941619.999280:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941619.999281:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941619.999287:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941619.999289:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 62 buffers still posted 00000100:00000001:3.0:1773941619.999291:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941619.999293:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941619.999295:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941619.999297:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d1121ef78. 00000400:00000200:3.0:1773941619.999299:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d2027e000 00000400:00000010:3.0:1773941619.999302:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d2027e000. 00000100:00000001:3.0:1773941619.999304:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941619.999306:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941619.999311:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941619.999312:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 61 buffers still posted 00000100:00000001:3.0:1773941619.999314:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941619.999316:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941619.999317:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941619.999319:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d1121eec8. 00000400:00000200:3.0:1773941619.999321:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d08febd80 00000400:00000010:3.0:1773941619.999323:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d08febd80. 00000100:00000001:3.0:1773941619.999332:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941619.999334:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941619.999339:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941619.999341:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 60 buffers still posted 00000100:00000001:3.0:1773941619.999344:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941619.999346:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941619.999347:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941619.999349:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d1121e058. 00000400:00000200:3.0:1773941619.999351:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d08febb40 00000400:00000010:3.0:1773941619.999353:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d08febb40. 00000100:00000001:3.0:1773941619.999355:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941619.999357:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941619.999362:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941619.999363:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 59 buffers still posted 00000100:00000001:3.0:1773941619.999365:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941619.999367:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941619.999368:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941619.999370:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d1121e268. 00000400:00000200:3.0:1773941619.999372:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d08febc00 00000400:00000010:3.0:1773941619.999374:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d08febc00. 00000100:00000001:3.0:1773941619.999376:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941619.999377:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941619.999382:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941619.999384:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 58 buffers still posted 00000100:00000001:3.0:1773941619.999386:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941619.999388:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941619.999389:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941619.999391:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d1121e528. 00000400:00000200:3.0:1773941619.999393:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d08feb0c0 00000400:00000010:3.0:1773941619.999395:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d08feb0c0. 00000100:00000001:3.0:1773941619.999397:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941619.999398:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941619.999403:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941619.999405:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 57 buffers still posted 00000100:00000001:3.0:1773941619.999407:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941619.999408:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941619.999410:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941619.999411:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d1121e210. 00000400:00000200:3.0:1773941619.999419:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d08febf00 00000400:00000010:3.0:1773941619.999421:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d08febf00. 00000100:00000001:3.0:1773941619.999424:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941619.999426:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941619.999431:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941619.999433:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 56 buffers still posted 00000100:00000001:3.0:1773941619.999435:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941619.999437:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941619.999438:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941619.999440:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d1121ebb0. 00000400:00000200:3.0:1773941619.999442:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d08feb9c0 00000400:00000010:3.0:1773941619.999444:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d08feb9c0. 00000100:00000001:3.0:1773941619.999446:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941619.999447:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941619.999452:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941619.999454:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 55 buffers still posted 00000100:00000001:3.0:1773941619.999455:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941619.999457:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941619.999458:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941619.999460:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d1121e420. 00000400:00000200:3.0:1773941619.999462:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d08feb240 00000400:00000010:3.0:1773941619.999464:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d08feb240. 00000100:00000001:3.0:1773941619.999466:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941619.999467:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941619.999472:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941619.999474:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 54 buffers still posted 00000100:00000001:3.0:1773941619.999476:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941619.999477:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941619.999479:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941619.999480:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d1121e790. 00000400:00000200:3.0:1773941619.999483:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d08feba80 00000400:00000010:3.0:1773941619.999484:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d08feba80. 00000100:00000001:3.0:1773941619.999487:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941619.999488:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941619.999493:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941619.999495:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 53 buffers still posted 00000100:00000001:3.0:1773941619.999496:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941619.999504:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941619.999505:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941619.999507:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d1121e370. 00000400:00000200:3.0:1773941619.999509:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d08febe40 00000400:00000010:3.0:1773941619.999511:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d08febe40. 00000100:00000001:3.0:1773941619.999513:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941619.999515:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941619.999520:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941619.999522:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 52 buffers still posted 00000100:00000001:3.0:1773941619.999523:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941619.999525:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941619.999526:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941619.999528:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d1121e840. 00000400:00000200:3.0:1773941619.999530:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d08feb6c0 00000400:00000010:3.0:1773941619.999532:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d08feb6c0. 00000100:00000001:3.0:1773941619.999534:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941619.999535:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941619.999540:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941619.999542:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 51 buffers still posted 00000100:00000001:3.0:1773941619.999543:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941619.999545:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941619.999547:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941619.999549:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d1121e7e8. 00000400:00000200:3.0:1773941619.999554:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d08feb480 00000400:00000010:3.0:1773941619.999557:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d08feb480. 00000100:00000001:3.0:1773941619.999561:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941619.999563:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941619.999571:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941619.999574:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 50 buffers still posted 00000100:00000001:3.0:1773941619.999577:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941619.999579:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941619.999581:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941619.999584:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d1121e5d8. 00000400:00000200:3.0:1773941619.999588:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d08fd3f00 00000400:00000010:3.0:1773941619.999592:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d08fd3f00. 00000100:00000001:3.0:1773941619.999597:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941619.999600:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941619.999608:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941619.999621:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 49 buffers still posted 00000100:00000001:3.0:1773941619.999624:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941619.999629:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941619.999631:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941619.999634:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d1121e580. 00000400:00000200:3.0:1773941619.999639:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d08fd39c0 00000400:00000010:3.0:1773941619.999643:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d08fd39c0. 00000100:00000001:3.0:1773941619.999647:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941619.999649:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941619.999658:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941619.999662:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 48 buffers still posted 00000100:00000001:3.0:1773941619.999664:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941619.999667:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941619.999668:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941619.999670:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d1121ed10. 00000400:00000200:3.0:1773941619.999673:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d08fd3180 00000400:00000010:3.0:1773941619.999676:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d08fd3180. 00000100:00000001:3.0:1773941619.999678:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941619.999680:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941619.999685:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941619.999686:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 47 buffers still posted 00000100:00000001:3.0:1773941619.999688:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941619.999690:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941619.999691:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941619.999693:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d1121e160. 00000400:00000200:3.0:1773941619.999695:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d08e343c0 00000400:00000010:3.0:1773941619.999697:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d08e343c0. 00000100:00000001:3.0:1773941619.999699:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941619.999701:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941619.999706:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941619.999708:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 46 buffers still posted 00000100:00000001:3.0:1773941619.999709:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941619.999711:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941619.999712:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941619.999714:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d1121eaa8. 00000400:00000200:3.0:1773941619.999716:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d08e34840 00000400:00000010:3.0:1773941619.999718:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d08e34840. 00000100:00000001:3.0:1773941619.999720:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941619.999729:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941619.999734:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941619.999737:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 45 buffers still posted 00000100:00000001:3.0:1773941619.999739:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941619.999742:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941619.999743:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941619.999745:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f580. 00000400:00000200:3.0:1773941619.999748:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d08e34cc0 00000400:00000010:3.0:1773941619.999750:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d08e34cc0. 00000100:00000001:3.0:1773941619.999752:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941619.999753:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941619.999758:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941619.999760:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 44 buffers still posted 00000100:00000001:3.0:1773941619.999762:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941619.999764:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941619.999766:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941619.999768:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f5d8. 00000400:00000200:3.0:1773941619.999770:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d08e34f00 00000400:00000010:3.0:1773941619.999772:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d08e34f00. 00000100:00000001:3.0:1773941619.999775:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941619.999776:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941619.999781:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941619.999783:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 43 buffers still posted 00000100:00000001:3.0:1773941619.999784:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941619.999786:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941619.999788:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941619.999789:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f840. 00000400:00000200:3.0:1773941619.999791:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d08e34900 00000400:00000010:3.0:1773941619.999793:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d08e34900. 00000100:00000001:3.0:1773941619.999795:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941619.999797:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941619.999802:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941619.999804:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 42 buffers still posted 00000100:00000001:3.0:1773941619.999805:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941619.999807:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941619.999808:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941619.999810:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f7e8. 00000400:00000200:3.0:1773941619.999812:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d08e34e40 00000400:00000010:3.0:1773941619.999821:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d08e34e40. 00000100:00000001:3.0:1773941619.999824:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941619.999826:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941619.999831:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941619.999833:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 41 buffers still posted 00000100:00000001:3.0:1773941619.999835:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941619.999837:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941619.999839:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941619.999840:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f370. 00000400:00000200:3.0:1773941619.999843:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d08e34300 00000400:00000010:3.0:1773941619.999844:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d08e34300. 00000100:00000001:3.0:1773941619.999847:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941619.999848:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941619.999853:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941619.999855:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 40 buffers still posted 00000100:00000001:3.0:1773941619.999856:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941619.999858:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941619.999859:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941619.999861:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037fd10. 00000400:00000200:3.0:1773941619.999863:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d08e34600 00000400:00000010:3.0:1773941619.999865:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d08e34600. 00000100:00000001:3.0:1773941619.999868:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941619.999869:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941619.999874:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941619.999876:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 39 buffers still posted 00000100:00000001:3.0:1773941619.999877:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941619.999879:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941619.999880:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941619.999882:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f160. 00000400:00000200:3.0:1773941619.999884:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d063e4540 00000400:00000010:3.0:1773941619.999886:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d063e4540. 00000100:00000001:3.0:1773941619.999889:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941619.999890:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941619.999895:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941619.999897:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 38 buffers still posted 00000100:00000001:3.0:1773941619.999898:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941619.999900:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941619.999906:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941619.999909:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037faa8. 00000400:00000200:3.0:1773941619.999911:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d063e46c0 00000400:00000010:3.0:1773941619.999913:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d063e46c0. 00000100:00000001:3.0:1773941619.999916:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941619.999945:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941619.999952:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941619.999954:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 37 buffers still posted 00000100:00000001:3.0:1773941619.999956:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.000047:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.000051:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.000054:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f108. 00000400:00000200:3.0:1773941620.000057:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d063e4d80 00000400:00000010:3.0:1773941620.000061:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d063e4d80. 00000100:00000001:3.0:1773941620.000066:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.000069:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.000081:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.000084:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 36 buffers still posted 00000100:00000001:3.0:1773941620.000091:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.000095:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.000099:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.000103:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037fa50. 00000400:00000200:3.0:1773941620.000111:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d063e4840 00000400:00000010:3.0:1773941620.000114:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d063e4840. 00000100:00000001:3.0:1773941620.000123:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.000126:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.000138:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.000146:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 35 buffers still posted 00000100:00000001:3.0:1773941620.000149:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.000152:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.000156:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.000160:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037ff78. 00000400:00000200:3.0:1773941620.000166:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d063e4300 00000400:00000010:3.0:1773941620.000169:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d063e4300. 00000100:00000001:3.0:1773941620.000175:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.000179:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.000185:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.000187:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 34 buffers still posted 00000100:00000001:3.0:1773941620.000198:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.000204:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.000207:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.000211:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f790. 00000400:00000200:3.0:1773941620.000215:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a29d9c0 00000400:00000010:3.0:1773941620.000217:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a29d9c0. 00000100:00000001:3.0:1773941620.000221:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.000222:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.000228:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.000230:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 33 buffers still posted 00000100:00000001:3.0:1773941620.000231:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.000233:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.000234:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.000236:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f420. 00000400:00000200:3.0:1773941620.000239:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a29d900 00000400:00000010:3.0:1773941620.000241:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a29d900. 00000100:00000001:3.0:1773941620.000244:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.000246:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.000259:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.000261:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 32 buffers still posted 00000100:00000001:3.0:1773941620.000263:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.000265:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.000266:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.000268:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f688. 00000400:00000200:3.0:1773941620.000271:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a29d300 00000400:00000010:3.0:1773941620.000273:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a29d300. 00000100:00000001:3.0:1773941620.000276:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.000278:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.000283:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.000286:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 31 buffers still posted 00000100:00000001:3.0:1773941620.000289:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.000291:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.000292:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.000295:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f478. 00000400:00000200:3.0:1773941620.000298:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a29dcc0 00000400:00000010:3.0:1773941620.000300:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a29dcc0. 00000100:00000001:3.0:1773941620.000308:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.000310:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.000325:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.000329:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 30 buffers still posted 00000100:00000001:3.0:1773941620.000333:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.000338:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.000341:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.000344:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f738. 00000400:00000200:3.0:1773941620.000350:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a29d0c0 00000400:00000010:3.0:1773941620.000353:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a29d0c0. 00000100:00000001:3.0:1773941620.000357:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.000358:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.000363:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.000365:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 29 buffers still posted 00000100:00000001:3.0:1773941620.000367:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.000368:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.000370:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.000372:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f0b0. 00000400:00000200:3.0:1773941620.000374:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a29df00 00000400:00000010:3.0:1773941620.000376:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a29df00. 00000100:00000001:3.0:1773941620.000378:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.000379:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.000384:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.000386:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 28 buffers still posted 00000100:00000001:3.0:1773941620.000388:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.000389:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.000391:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.000393:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037fdc0. 00000400:00000200:3.0:1773941620.000395:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a29d480 00000400:00000010:3.0:1773941620.000397:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a29d480. 00000100:00000001:3.0:1773941620.000400:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.000401:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.000406:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.000408:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 27 buffers still posted 00000100:00000001:3.0:1773941620.000410:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.000411:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.000413:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.000414:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f3c8. 00000400:00000200:3.0:1773941620.000416:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a033cc0 00000400:00000010:3.0:1773941620.000418:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a033cc0. 00000100:00000001:3.0:1773941620.000429:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.000430:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.000435:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.000437:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 26 buffers still posted 00000100:00000001:3.0:1773941620.000439:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.000441:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.000442:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.000444:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f8f0. 00000400:00000200:3.0:1773941620.000446:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a0333c0 00000400:00000010:3.0:1773941620.000448:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a0333c0. 00000100:00000001:3.0:1773941620.000450:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.000452:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.000457:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.000458:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 25 buffers still posted 00000100:00000001:3.0:1773941620.000460:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.000462:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.000463:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.000465:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037fd68. 00000400:00000200:3.0:1773941620.000467:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a03c840 00000400:00000010:3.0:1773941620.000469:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a03c840. 00000100:00000001:3.0:1773941620.000471:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.000473:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.000477:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.000479:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 24 buffers still posted 00000100:00000001:3.0:1773941620.000481:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.000483:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.000484:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.000486:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037ff20. 00000400:00000200:3.0:1773941620.000488:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a03c240 00000400:00000010:3.0:1773941620.000490:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a03c240. 00000100:00000001:3.0:1773941620.000492:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.000493:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.000498:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.000500:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 23 buffers still posted 00000100:00000001:3.0:1773941620.000501:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.000503:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.000505:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.000506:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037fb58. 00000400:00000200:3.0:1773941620.000514:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a03c600 00000400:00000010:3.0:1773941620.000516:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a03c600. 00000100:00000001:3.0:1773941620.000518:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.000520:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.000525:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.000528:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 22 buffers still posted 00000100:00000001:3.0:1773941620.000529:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.000531:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.000533:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.000534:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f6e0. 00000400:00000200:3.0:1773941620.000536:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a03ccc0 00000400:00000010:3.0:1773941620.000538:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a03ccc0. 00000100:00000001:3.0:1773941620.000540:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.000542:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.000548:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.000551:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 21 buffers still posted 00000100:00000001:3.0:1773941620.000554:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.000556:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.000558:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.000561:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037fcb8. 00000400:00000200:3.0:1773941620.000565:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a03c180 00000400:00000010:3.0:1773941620.000568:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a03c180. 00000100:00000001:3.0:1773941620.000572:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.000574:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.000582:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.000585:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 20 buffers still posted 00000100:00000001:3.0:1773941620.000588:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.000591:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.000594:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.000597:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f000. 00000400:00000200:3.0:1773941620.000601:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a03cf00 00000400:00000010:3.0:1773941620.000605:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a03cf00. 00000100:00000001:3.0:1773941620.000609:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.000611:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.000619:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.000623:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 19 buffers still posted 00000100:00000001:3.0:1773941620.000626:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.000641:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.000643:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.000647:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037fc60. 00000400:00000200:3.0:1773941620.000652:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d20863780 00000400:00000010:3.0:1773941620.000656:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d20863780. 00000100:00000001:3.0:1773941620.000660:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.000663:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.000672:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.000675:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 18 buffers still posted 00000100:00000001:3.0:1773941620.000679:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.000684:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.000685:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.000688:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f1b8. 00000400:00000200:3.0:1773941620.000690:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d20863f00 00000400:00000010:3.0:1773941620.000692:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d20863f00. 00000100:00000001:3.0:1773941620.000695:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.000697:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.000702:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.000704:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 17 buffers still posted 00000100:00000001:3.0:1773941620.000706:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.000707:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.000709:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.000710:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037fe18. 00000400:00000200:3.0:1773941620.000713:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d208639c0 00000400:00000010:3.0:1773941620.000714:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d208639c0. 00000100:00000001:3.0:1773941620.000717:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.000718:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.000723:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.000725:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 16 buffers still posted 00000100:00000001:3.0:1773941620.000727:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.000728:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.000730:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.000731:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037fb00. 00000400:00000200:3.0:1773941620.000733:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d20863480 00000400:00000010:3.0:1773941620.000735:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d20863480. 00000100:00000001:3.0:1773941620.000737:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.000739:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.000743:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.000754:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 15 buffers still posted 00000100:00000001:3.0:1773941620.000756:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.000759:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.000760:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.000762:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f898. 00000400:00000200:3.0:1773941620.000765:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d20863900 00000400:00000010:3.0:1773941620.000767:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d20863900. 00000100:00000001:3.0:1773941620.000769:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.000770:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.000775:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.000777:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 14 buffers still posted 00000100:00000001:3.0:1773941620.000779:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.000781:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.000782:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.000784:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037fe70. 00000400:00000200:3.0:1773941620.000786:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d20863540 00000400:00000010:3.0:1773941620.000788:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d20863540. 00000100:00000001:3.0:1773941620.000790:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.000791:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.000796:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.000798:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 13 buffers still posted 00000100:00000001:3.0:1773941620.000800:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.000801:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.000803:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.000804:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f318. 00000400:00000200:3.0:1773941620.000807:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d20863a80 00000400:00000010:3.0:1773941620.000809:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d20863a80. 00000100:00000001:3.0:1773941620.000811:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.000813:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.000818:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.000820:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 12 buffers still posted 00000100:00000001:3.0:1773941620.000821:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.000823:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.000824:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.000826:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f9f8. 00000400:00000200:3.0:1773941620.000828:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d20863e40 00000400:00000010:3.0:1773941620.000830:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d20863e40. 00000100:00000001:3.0:1773941620.000832:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.000839:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.000844:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.000846:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 11 buffers still posted 00000100:00000001:3.0:1773941620.000848:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.000851:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.000853:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.000855:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f630. 00000400:00000200:3.0:1773941620.000857:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d20863840 00000400:00000010:3.0:1773941620.000859:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d20863840. 00000100:00000001:3.0:1773941620.000861:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.000862:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.000867:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.000869:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 10 buffers still posted 00000100:00000001:3.0:1773941620.000871:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.000873:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.000874:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.000876:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037fc08. 00000400:00000200:3.0:1773941620.000878:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d20863180 00000400:00000010:3.0:1773941620.000880:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d20863180. 00000100:00000001:3.0:1773941620.000882:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.000883:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.000888:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.000890:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 9 buffers still posted 00000100:00000001:3.0:1773941620.000892:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.000893:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.000895:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.000896:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f4d0. 00000400:00000200:3.0:1773941620.000898:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d20863300 00000400:00000010:3.0:1773941620.000900:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d20863300. 00000100:00000001:3.0:1773941620.000902:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.000904:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.000909:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.000910:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 8 buffers still posted 00000100:00000001:3.0:1773941620.000912:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.000914:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.000915:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.000917:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f9a0. 00000400:00000200:3.0:1773941620.000939:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04c810c0 00000400:00000010:3.0:1773941620.000951:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d04c810c0. 00000100:00000001:3.0:1773941620.000956:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.000957:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001001:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001008:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 7 buffers still posted 00000100:00000001:3.0:1773941620.001015:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001019:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001021:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001025:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f948. 00000400:00000200:3.0:1773941620.001032:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d04c81840 00000400:00000010:3.0:1773941620.001035:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d04c81840. 00000100:00000001:3.0:1773941620.001043:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001046:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001052:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001056:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 6 buffers still posted 00000100:00000001:3.0:1773941620.001060:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001062:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001064:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001066:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f2c0. 00000400:00000200:3.0:1773941620.001069:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d034cc240 00000400:00000010:3.0:1773941620.001072:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d034cc240. 00000100:00000001:3.0:1773941620.001075:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001077:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001082:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001084:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 5 buffers still posted 00000100:00000001:3.0:1773941620.001085:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001087:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001088:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001090:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037fec8. 00000400:00000200:3.0:1773941620.001092:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d2081ad80 00000400:00000010:3.0:1773941620.001094:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d2081ad80. 00000100:00000001:3.0:1773941620.001097:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001098:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001103:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001105:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 4 buffers still posted 00000100:00000001:3.0:1773941620.001106:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001108:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001109:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001118:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f058. 00000400:00000200:3.0:1773941620.001121:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d2081a0c0 00000400:00000010:3.0:1773941620.001123:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d2081a0c0. 00000100:00000001:3.0:1773941620.001126:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001128:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001134:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001135:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 3 buffers still posted 00000100:00000001:3.0:1773941620.001137:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001139:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001140:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001142:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f268. 00000400:00000200:3.0:1773941620.001144:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d2081a480 00000400:00000010:3.0:1773941620.001146:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d2081a480. 00000100:00000001:3.0:1773941620.001148:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001150:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001155:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001157:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 2 buffers still posted 00000100:00000001:3.0:1773941620.001158:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001160:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001161:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001163:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f528. 00000400:00000200:3.0:1773941620.001165:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d06510780 00000400:00000010:3.0:1773941620.001167:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d06510780. 00000100:00000001:3.0:1773941620.001169:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001171:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001176:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001177:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 1 buffers still posted 00000100:00000001:3.0:1773941620.001179:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001181:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001182:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001184:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d2037f210. 00000400:00000200:3.0:1773941620.001186:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d06510b40 00000400:00000010:3.0:1773941620.001188:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d06510b40. 00000100:00000001:3.0:1773941620.001190:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001191:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001196:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001198:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 0 buffers still posted 00000100:00000001:3.0:1773941620.001200:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001208:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001209:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001212:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379d10. 00000400:00000200:3.0:1773941620.001215:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d09350240 00000400:00000010:3.0:1773941620.001217:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d09350240. 00000100:00000001:3.0:1773941620.001220:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001221:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001226:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001228:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 63 buffers still posted 00000100:00000001:3.0:1773941620.001230:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001232:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001233:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001235:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379160. 00000400:00000200:3.0:1773941620.001237:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d09350f00 00000400:00000010:3.0:1773941620.001239:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d09350f00. 00000100:00000001:3.0:1773941620.001241:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001243:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001247:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001249:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 62 buffers still posted 00000100:00000001:3.0:1773941620.001251:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001253:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001254:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001256:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379aa8. 00000400:00000200:3.0:1773941620.001258:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d093503c0 00000400:00000010:3.0:1773941620.001260:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d093503c0. 00000100:00000001:3.0:1773941620.001262:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001263:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001268:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001270:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 61 buffers still posted 00000100:00000001:3.0:1773941620.001271:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001273:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001275:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001276:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379108. 00000400:00000200:3.0:1773941620.001278:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d093500c0 00000400:00000010:3.0:1773941620.001280:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d093500c0. 00000100:00000001:3.0:1773941620.001282:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001284:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001289:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001295:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 60 buffers still posted 00000100:00000001:3.0:1773941620.001297:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001300:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001301:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001304:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379a50. 00000400:00000200:3.0:1773941620.001306:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d09350300 00000400:00000010:3.0:1773941620.001308:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d09350300. 00000100:00000001:3.0:1773941620.001310:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001312:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001316:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001318:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 59 buffers still posted 00000100:00000001:3.0:1773941620.001320:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001321:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001323:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001324:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379f78. 00000400:00000200:3.0:1773941620.001327:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d09350c00 00000400:00000010:3.0:1773941620.001328:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d09350c00. 00000100:00000001:3.0:1773941620.001330:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001332:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001337:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001338:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 58 buffers still posted 00000100:00000001:3.0:1773941620.001340:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001341:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001343:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001344:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379688. 00000400:00000200:3.0:1773941620.001347:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d09350d80 00000400:00000010:3.0:1773941620.001348:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d09350d80. 00000100:00000001:3.0:1773941620.001350:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001352:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001356:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001358:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 57 buffers still posted 00000100:00000001:3.0:1773941620.001360:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001361:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001363:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001364:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379478. 00000400:00000200:3.0:1773941620.001367:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d09350e40 00000400:00000010:3.0:1773941620.001368:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d09350e40. 00000100:00000001:3.0:1773941620.001370:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001377:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001382:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001385:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 56 buffers still posted 00000100:00000001:3.0:1773941620.001387:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001390:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001391:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001393:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379738. 00000400:00000200:3.0:1773941620.001396:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d09350900 00000400:00000010:3.0:1773941620.001397:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d09350900. 00000100:00000001:3.0:1773941620.001399:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001401:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001406:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001407:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 55 buffers still posted 00000100:00000001:3.0:1773941620.001409:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001411:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001412:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001414:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203790b0. 00000400:00000200:3.0:1773941620.001416:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d09350000 00000400:00000010:3.0:1773941620.001418:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d09350000. 00000100:00000001:3.0:1773941620.001420:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001421:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001426:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001428:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 54 buffers still posted 00000100:00000001:3.0:1773941620.001429:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001431:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001432:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001434:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379dc0. 00000400:00000200:3.0:1773941620.001436:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d09350600 00000400:00000010:3.0:1773941620.001438:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d09350600. 00000100:00000001:3.0:1773941620.001440:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001441:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001446:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001448:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 53 buffers still posted 00000100:00000001:3.0:1773941620.001450:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001451:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001452:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001454:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203793c8. 00000400:00000200:3.0:1773941620.001456:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d09350a80 00000400:00000010:3.0:1773941620.001465:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d09350a80. 00000100:00000001:3.0:1773941620.001468:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001469:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001474:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001476:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 52 buffers still posted 00000100:00000001:3.0:1773941620.001478:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001479:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001481:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001482:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d203798f0. 00000400:00000200:3.0:1773941620.001485:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d21c4ccc0 00000400:00000010:3.0:1773941620.001487:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d21c4ccc0. 00000100:00000001:3.0:1773941620.001489:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001491:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001495:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001497:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 51 buffers still posted 00000100:00000001:3.0:1773941620.001499:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001500:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001502:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001503:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379d68. 00000400:00000200:3.0:1773941620.001505:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d21c4c0c0 00000400:00000010:3.0:1773941620.001507:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d21c4c0c0. 00000100:00000001:3.0:1773941620.001509:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001511:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001516:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001517:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 50 buffers still posted 00000100:00000001:3.0:1773941620.001519:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001521:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001522:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001524:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379f20. 00000400:00000200:3.0:1773941620.001526:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d21c4c480 00000400:00000010:3.0:1773941620.001527:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d21c4c480. 00000100:00000001:3.0:1773941620.001530:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001531:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001536:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001537:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 49 buffers still posted 00000100:00000001:3.0:1773941620.001539:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001541:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001549:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001552:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379b58. 00000400:00000200:3.0:1773941620.001556:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d21c4c600 00000400:00000010:3.0:1773941620.001559:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d21c4c600. 00000100:00000001:3.0:1773941620.001564:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001566:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001575:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001578:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 48 buffers still posted 00000100:00000001:3.0:1773941620.001581:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001584:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001587:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001590:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379cb8. 00000400:00000200:3.0:1773941620.001595:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d21c4cd80 00000400:00000010:3.0:1773941620.001598:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d21c4cd80. 00000100:00000001:3.0:1773941620.001601:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001604:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001611:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001615:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 47 buffers still posted 00000100:00000001:3.0:1773941620.001618:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001621:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001623:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001627:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d20379000. 00000400:00000200:3.0:1773941620.001631:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d21c4ce40 00000400:00000010:3.0:1773941620.001635:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d21c4ce40. 00000100:00000001:3.0:1773941620.001639:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001641:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001646:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001649:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 46 buffers still posted 00000100:00000001:3.0:1773941620.001651:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001653:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001654:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001656:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e058. 00000400:00000200:3.0:1773941620.001659:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d21c4c540 00000400:00000010:3.0:1773941620.001661:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d21c4c540. 00000100:00000001:3.0:1773941620.001663:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001665:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001669:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001671:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 45 buffers still posted 00000100:00000001:3.0:1773941620.001681:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001685:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001687:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001689:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380eec8. 00000400:00000200:3.0:1773941620.001692:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d21c4c240 00000400:00000010:3.0:1773941620.001694:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d21c4c240. 00000100:00000001:3.0:1773941620.001697:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001698:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001704:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001706:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 44 buffers still posted 00000100:00000001:3.0:1773941620.001708:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001710:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001711:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001713:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e2c0. 00000400:00000200:3.0:1773941620.001715:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d21c4cc00 00000400:00000010:3.0:1773941620.001717:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d21c4cc00. 00000100:00000001:3.0:1773941620.001719:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001720:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001725:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001727:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 43 buffers still posted 00000100:00000001:3.0:1773941620.001729:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001730:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001732:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001733:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e9a0. 00000400:00000200:3.0:1773941620.001735:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d21c4c900 00000400:00000010:3.0:1773941620.001737:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d21c4c900. 00000100:00000001:3.0:1773941620.001739:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001741:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001746:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001748:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 42 buffers still posted 00000100:00000001:3.0:1773941620.001749:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001751:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001752:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001754:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e268. 00000400:00000200:3.0:1773941620.001756:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d21c4c180 00000400:00000010:3.0:1773941620.001758:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d21c4c180. 00000100:00000001:3.0:1773941620.001760:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001761:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001771:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001774:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 41 buffers still posted 00000100:00000001:3.0:1773941620.001776:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001778:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001780:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001782:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e528. 00000400:00000200:3.0:1773941620.001785:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d21c4cf00 00000400:00000010:3.0:1773941620.001786:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d21c4cf00. 00000100:00000001:3.0:1773941620.001789:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001790:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001795:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001797:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 40 buffers still posted 00000100:00000001:3.0:1773941620.001798:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001800:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001801:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001803:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e210. 00000400:00000200:3.0:1773941620.001805:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d21c4ca80 00000400:00000010:3.0:1773941620.001807:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d21c4ca80. 00000100:00000001:3.0:1773941620.001809:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001810:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001815:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001817:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 39 buffers still posted 00000100:00000001:3.0:1773941620.001818:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001820:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001821:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001823:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380ebb0. 00000400:00000200:3.0:1773941620.001825:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d21c4c840 00000400:00000010:3.0:1773941620.001827:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d21c4c840. 00000100:00000001:3.0:1773941620.001829:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001830:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001835:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001837:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 38 buffers still posted 00000100:00000001:3.0:1773941620.001839:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001840:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001842:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001843:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e420. 00000400:00000200:3.0:1773941620.001846:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d21c4c3c0 00000400:00000010:3.0:1773941620.001847:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d21c4c3c0. 00000100:00000001:3.0:1773941620.001855:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001857:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001862:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001864:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 37 buffers still posted 00000100:00000001:3.0:1773941620.001867:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001869:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001870:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001872:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e790. 00000400:00000200:3.0:1773941620.001874:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d21c4c780 00000400:00000010:3.0:1773941620.001876:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d21c4c780. 00000100:00000001:3.0:1773941620.001878:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001880:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001885:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001886:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 36 buffers still posted 00000100:00000001:3.0:1773941620.001888:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001890:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001891:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001893:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e370. 00000400:00000200:3.0:1773941620.001895:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d21c4c000 00000400:00000010:3.0:1773941620.001896:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d21c4c000. 00000100:00000001:3.0:1773941620.001899:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001900:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001905:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001907:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 35 buffers still posted 00000100:00000001:3.0:1773941620.001908:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001910:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001911:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001913:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e840. 00000400:00000200:3.0:1773941620.001915:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d21c4c6c0 00000400:00000010:3.0:1773941620.001917:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d21c4c6c0. 00000100:00000001:3.0:1773941620.001940:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.001942:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.001948:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.001950:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 34 buffers still posted 00000100:00000001:3.0:1773941620.001952:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.001954:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.001955:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.001957:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e7e8. 00000400:00000200:3.0:1773941620.001991:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d21c4c300 00000400:00000010:3.0:1773941620.001994:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d21c4c300. 00000100:00000001:3.0:1773941620.001998:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.002000:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.002014:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.002017:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 33 buffers still posted 00000100:00000001:3.0:1773941620.002029:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.002033:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.002034:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.002037:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e5d8. 00000400:00000200:3.0:1773941620.002041:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18807840 00000400:00000010:3.0:1773941620.002043:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18807840. 00000100:00000001:3.0:1773941620.002046:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.002048:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.002053:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.002054:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 32 buffers still posted 00000100:00000001:3.0:1773941620.002056:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.002058:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.002059:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.002061:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e580. 00000400:00000200:3.0:1773941620.002063:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18807300 00000400:00000010:3.0:1773941620.002065:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18807300. 00000100:00000001:3.0:1773941620.002067:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.002069:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.002074:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.002075:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 31 buffers still posted 00000100:00000001:3.0:1773941620.002077:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.002079:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.002080:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.002082:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380ed10. 00000400:00000200:3.0:1773941620.002084:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d188070c0 00000400:00000010:3.0:1773941620.002086:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d188070c0. 00000100:00000001:3.0:1773941620.002088:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.002089:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.002094:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.002096:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 30 buffers still posted 00000100:00000001:3.0:1773941620.002097:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.002105:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.002107:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.002109:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e160. 00000400:00000200:3.0:1773941620.002112:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18807cc0 00000400:00000010:3.0:1773941620.002114:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18807cc0. 00000100:00000001:3.0:1773941620.002117:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.002119:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.002124:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.002126:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 29 buffers still posted 00000100:00000001:3.0:1773941620.002127:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.002129:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.002131:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.002133:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380eaa8. 00000400:00000200:3.0:1773941620.002135:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18807780 00000400:00000010:3.0:1773941620.002137:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18807780. 00000100:00000001:3.0:1773941620.002139:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.002140:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.002145:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.002147:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 28 buffers still posted 00000100:00000001:3.0:1773941620.002149:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.002150:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.002151:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.002153:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e108. 00000400:00000200:3.0:1773941620.002155:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18807000 00000400:00000010:3.0:1773941620.002157:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18807000. 00000100:00000001:3.0:1773941620.002159:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.002161:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.002165:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.002167:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 27 buffers still posted 00000100:00000001:3.0:1773941620.002169:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.002170:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.002172:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.002173:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380ea50. 00000400:00000200:3.0:1773941620.002175:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18807240 00000400:00000010:3.0:1773941620.002177:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18807240. 00000100:00000001:3.0:1773941620.002179:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.002181:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.002186:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.002194:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 26 buffers still posted 00000100:00000001:3.0:1773941620.002196:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.002199:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.002200:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.002202:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380ef78. 00000400:00000200:3.0:1773941620.002205:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18807e40 00000400:00000010:3.0:1773941620.002207:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18807e40. 00000100:00000001:3.0:1773941620.002209:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.002210:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.002215:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.002217:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 25 buffers still posted 00000100:00000001:3.0:1773941620.002219:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.002220:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.002222:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.002223:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e688. 00000400:00000200:3.0:1773941620.002225:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18807600 00000400:00000010:3.0:1773941620.002227:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18807600. 00000100:00000001:3.0:1773941620.002229:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.002231:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.002236:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.002237:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 24 buffers still posted 00000100:00000001:3.0:1773941620.002239:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.002240:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.002242:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.002243:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e478. 00000400:00000200:3.0:1773941620.002245:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18807480 00000400:00000010:3.0:1773941620.002247:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18807480. 00000100:00000001:3.0:1773941620.002249:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.002251:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.002255:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.002257:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 23 buffers still posted 00000100:00000001:3.0:1773941620.002259:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.002260:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.002262:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.002263:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e738. 00000400:00000200:3.0:1773941620.002265:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d188079c0 00000400:00000010:3.0:1773941620.002267:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d188079c0. 00000100:00000001:3.0:1773941620.002269:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.002275:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.002281:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.002283:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 22 buffers still posted 00000100:00000001:3.0:1773941620.002285:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.002287:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.002289:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.002290:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e0b0. 00000400:00000200:3.0:1773941620.002293:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18807180 00000400:00000010:3.0:1773941620.002295:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18807180. 00000100:00000001:3.0:1773941620.002297:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.002298:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.002303:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.002305:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 21 buffers still posted 00000100:00000001:3.0:1773941620.002307:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.002308:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.002310:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.002311:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380edc0. 00000400:00000200:3.0:1773941620.002313:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d188073c0 00000400:00000010:3.0:1773941620.002315:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d188073c0. 00000100:00000001:3.0:1773941620.002317:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.002319:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.002323:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.002325:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 20 buffers still posted 00000100:00000001:3.0:1773941620.002327:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.002328:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.002329:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.002331:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e3c8. 00000400:00000200:3.0:1773941620.002333:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18807a80 00000400:00000010:3.0:1773941620.002335:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18807a80. 00000100:00000001:3.0:1773941620.002337:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.002339:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.002343:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.002345:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 19 buffers still posted 00000100:00000001:3.0:1773941620.002347:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.002348:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.002349:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.002351:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e8f0. 00000400:00000200:3.0:1773941620.002353:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18807c00 00000400:00000010:3.0:1773941620.002360:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18807c00. 00000100:00000001:3.0:1773941620.002363:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.002364:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.002370:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.002372:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 18 buffers still posted 00000100:00000001:3.0:1773941620.002373:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.002375:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.002376:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.002378:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380ed68. 00000400:00000200:3.0:1773941620.002380:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18807d80 00000400:00000010:3.0:1773941620.002382:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18807d80. 00000100:00000001:3.0:1773941620.002384:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.002386:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.002390:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.002392:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 17 buffers still posted 00000100:00000001:3.0:1773941620.002394:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.002396:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.002397:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.002399:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380ef20. 00000400:00000200:3.0:1773941620.002401:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d18807900 00000400:00000010:3.0:1773941620.002403:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d18807900. 00000100:00000001:3.0:1773941620.002405:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.002406:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.002411:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.002413:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 16 buffers still posted 00000100:00000001:3.0:1773941620.002415:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.002416:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.002417:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.002419:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380eb58. 00000400:00000200:3.0:1773941620.002421:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d035dac00 00000400:00000010:3.0:1773941620.002423:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d035dac00. 00000100:00000001:3.0:1773941620.002426:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.002427:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.002432:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.002434:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 15 buffers still posted 00000100:00000001:3.0:1773941620.002435:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.002437:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.002443:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.002445:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e6e0. 00000400:00000200:3.0:1773941620.002448:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d035daf00 00000400:00000010:3.0:1773941620.002449:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d035daf00. 00000100:00000001:3.0:1773941620.002452:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.002454:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.002459:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.002460:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 14 buffers still posted 00000100:00000001:3.0:1773941620.002462:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.002464:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.002465:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.002467:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380ecb8. 00000400:00000200:3.0:1773941620.002469:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d035da0c0 00000400:00000010:3.0:1773941620.002471:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d035da0c0. 00000100:00000001:3.0:1773941620.002473:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.002474:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.002479:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.002481:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 13 buffers still posted 00000100:00000001:3.0:1773941620.002482:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.002484:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.002485:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.002487:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e000. 00000400:00000200:3.0:1773941620.002489:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d035da840 00000400:00000010:3.0:1773941620.002491:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d035da840. 00000100:00000001:3.0:1773941620.002493:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.002494:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.002499:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.002501:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 12 buffers still posted 00000100:00000001:3.0:1773941620.002503:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.002504:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.002505:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.002507:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380ec60. 00000400:00000200:3.0:1773941620.002509:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d035da480 00000400:00000010:3.0:1773941620.002511:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d035da480. 00000100:00000001:3.0:1773941620.002513:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.002514:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.002519:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.002521:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 11 buffers still posted 00000100:00000001:3.0:1773941620.002527:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.002530:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.002531:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.002533:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e1b8. 00000400:00000200:3.0:1773941620.002535:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d035da300 00000400:00000010:3.0:1773941620.002537:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d035da300. 00000100:00000001:3.0:1773941620.002539:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.002540:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.002546:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.002549:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 10 buffers still posted 00000100:00000001:3.0:1773941620.002552:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.002554:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.002556:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.002559:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380ee18. 00000400:00000200:3.0:1773941620.002563:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d035dacc0 00000400:00000010:3.0:1773941620.002566:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d035dacc0. 00000100:00000001:3.0:1773941620.002570:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.002572:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.002580:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.002583:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 9 buffers still posted 00000100:00000001:3.0:1773941620.002586:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.002589:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.002591:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.002595:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380eb00. 00000400:00000200:3.0:1773941620.002599:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d035daa80 00000400:00000010:3.0:1773941620.002602:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d035daa80. 00000100:00000001:3.0:1773941620.002606:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.002609:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.002617:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.002620:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 8 buffers still posted 00000100:00000001:3.0:1773941620.002624:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.002626:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.002629:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.002632:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e898. 00000400:00000200:3.0:1773941620.002636:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d035dad80 00000400:00000010:3.0:1773941620.002639:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d035dad80. 00000100:00000001:3.0:1773941620.002641:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.002643:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.002657:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.002659:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 7 buffers still posted 00000100:00000001:3.0:1773941620.002661:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.002665:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.002666:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.002669:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380ee70. 00000400:00000200:3.0:1773941620.002671:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d0705b780 00000400:00000010:3.0:1773941620.002674:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d0705b780. 00000100:00000001:3.0:1773941620.002678:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.002679:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.002684:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.002686:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 6 buffers still posted 00000100:00000001:3.0:1773941620.002688:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.002689:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.002691:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.002692:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e318. 00000400:00000200:3.0:1773941620.002694:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d1a040840 00000400:00000010:3.0:1773941620.002696:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d1a040840. 00000100:00000001:3.0:1773941620.002699:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.002700:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.002706:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.002708:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 5 buffers still posted 00000100:00000001:3.0:1773941620.002710:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.002712:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.002713:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.002715:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e9f8. 00000400:00000200:3.0:1773941620.002717:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d02f7d900 00000400:00000010:3.0:1773941620.002719:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d02f7d900. 00000100:00000001:3.0:1773941620.002722:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.002723:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.002728:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.002730:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 4 buffers still posted 00000100:00000001:3.0:1773941620.002731:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.002733:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.002734:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.002736:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e630. 00000400:00000200:3.0:1773941620.002738:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d02f7dd80 00000400:00000010:3.0:1773941620.002740:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d02f7dd80. 00000100:00000001:3.0:1773941620.002748:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.002750:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.002756:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.002758:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 3 buffers still posted 00000100:00000001:3.0:1773941620.002760:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.002762:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.002763:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.002765:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380ec08. 00000400:00000200:3.0:1773941620.002767:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d02f7d300 00000400:00000010:3.0:1773941620.002769:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d02f7d300. 00000100:00000001:3.0:1773941620.002771:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.002773:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.002778:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.002780:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 2 buffers still posted 00000100:00000001:3.0:1773941620.002781:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.002783:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.002785:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.002786:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e4d0. 00000400:00000200:3.0:1773941620.002788:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d02f7da80 00000400:00000010:3.0:1773941620.002790:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d02f7da80. 00000100:00000001:3.0:1773941620.002792:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.002794:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.002799:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.002801:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 1 buffers still posted 00000100:00000001:3.0:1773941620.002802:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000400:00000001:3.0:1773941620.002804:0:14694:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1773941620.002805:0:14694:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000400:00000010:3.0:1773941620.002807:0:14694:0:(lib-me.c:113:lnet_me_unlink()) slab-freed 'me': 88 at ffff988d0380e948. 00000400:00000200:3.0:1773941620.002809:0:14694:0:(lib-md.c:46:lnet_md_unlink()) Unlinking md ffff988d02f7d3c0 00000400:00000010:3.0:1773941620.002811:0:14694:0:(lib-lnet.h:315:lnet_md_free()) kfreed 'md': 152 at ffff988d02f7d3c0. 00000100:00000001:3.0:1773941620.002813:0:14694:0:(events.c:286:request_in_callback()) Process entered 00000100:00000200:3.0:1773941620.002814:0:14694:0:(events.c:296:request_in_callback()) event type 6, status 0, service ldlm_canceld 00000100:00100000:3.0:1773941620.002819:0:14694:0:(events.c:340:request_in_callback()) peer: 0-0@<0:0> (source: 0-0@<0:0>) 00000100:00000040:3.0:1773941620.002821:0:14694:0:(events.c:349:request_in_callback()) Buffer complete: 0 buffers still posted 00000100:00000001:3.0:1773941620.002823:0:14694:0:(events.c:373:request_in_callback()) Process leaving 00000100:00000001:3.0:1773941620.002825:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.002827:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.002830:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.002838:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.002840:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.002842:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.002844:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.002845:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.002847:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.002849:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.002850:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.002852:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.002853:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.002855:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.002856:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.002857:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.002859:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.002860:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.002862:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.002863:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.002865:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.002866:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.002868:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.002869:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.002871:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.002872:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.002874:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.002875:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.002877:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.002878:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.002879:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.002881:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.002882:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.002884:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.002885:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.002886:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.002888:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.002889:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.002891:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.002892:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.002894:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.002895:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.002897:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.002903:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.002905:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.002907:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.002908:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.002910:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.002912:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.002914:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.002916:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.002917:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.002941:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.002944:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.002945:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.002947:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.002948:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.002950:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.002951:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.002953:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.002955:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.002957:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.002983:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.002985:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.002987:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.002989:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.002991:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.002992:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.002994:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.002995:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.002997:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003001:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003005:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003015:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003017:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003020:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003027:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003029:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003031:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003034:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003036:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003038:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003040:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003049:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003055:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003058:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003062:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003065:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003068:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003070:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003072:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003075:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003078:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003079:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003081:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003083:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003084:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003086:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003089:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003091:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003095:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003097:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003101:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003103:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003106:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003107:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003110:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003112:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003114:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003115:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003119:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003121:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003124:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003125:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003127:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003128:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003130:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003131:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003133:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003134:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003135:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003137:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003139:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003145:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003147:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003149:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003150:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003152:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000010:3.0:1773941620.003157:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d10800000. 00000100:00000010:3.0:1773941620.003163:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1081e800. 00000100:00000010:3.0:1773941620.003166:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d10804000. 00000100:00000010:3.0:1773941620.003170:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1081cc00. 00000100:00000010:3.0:1773941620.003173:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08380000. 00000100:00000010:3.0:1773941620.003178:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1081c400. 00000100:00000010:3.0:1773941620.003181:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08384000. 00000100:00000010:3.0:1773941620.003187:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1081c000. 00000100:00000010:3.0:1773941620.003189:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08386000. 00000100:00000010:3.0:1773941620.003194:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1081ec00. 00000100:00000010:3.0:1773941620.003197:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d065e8000. 00000100:00000010:3.0:1773941620.003202:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1081f400. 00000100:00000010:3.0:1773941620.003205:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d065ea000. 00000100:00000010:3.0:1773941620.003210:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1081f000. 00000100:00000010:3.0:1773941620.003213:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d065ee000. 00000100:00000010:3.0:1773941620.003216:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1081d000. 00000100:00000010:3.0:1773941620.003219:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d065ec000. 00000100:00000010:3.0:1773941620.003223:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1081f800. 00000100:00000010:3.0:1773941620.003226:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d107fc000. 00000100:00000010:3.0:1773941620.003229:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d113e1800. 00000100:00000010:3.0:1773941620.003232:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d107f8000. 00000100:00000010:3.0:1773941620.003236:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d113e2c00. 00000100:00000010:3.0:1773941620.003238:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d107fa000. 00000100:00000010:3.0:1773941620.003242:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d113e3c00. 00000100:00000010:3.0:1773941620.003244:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d107fe000. 00000100:00000010:3.0:1773941620.003248:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d113e1000. 00000100:00000010:3.0:1773941620.003250:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d0337a000. 00000100:00000010:3.0:1773941620.003255:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d113e2000. 00000100:00000010:3.0:1773941620.003258:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d0337e000. 00000100:00000010:3.0:1773941620.003267:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d113e0800. 00000100:00000010:3.0:1773941620.003270:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d0337c000. 00000100:00000010:3.0:1773941620.003274:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d113e1400. 00000100:00000010:3.0:1773941620.003276:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d03378000. 00000100:00000010:3.0:1773941620.003280:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d113e3000. 00000100:00000010:3.0:1773941620.003282:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d07fd2000. 00000100:00000010:3.0:1773941620.003286:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d113e1c00. 00000100:00000010:3.0:1773941620.003288:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d07fd6000. 00000100:00000010:3.0:1773941620.003292:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d113e0c00. 00000100:00000010:3.0:1773941620.003294:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d07fd4000. 00000100:00000010:3.0:1773941620.003297:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d113e3800. 00000100:00000010:3.0:1773941620.003300:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d07fd0000. 00000100:00000010:3.0:1773941620.003303:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d113e2800. 00000100:00000010:3.0:1773941620.003305:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d20304000. 00000100:00000010:3.0:1773941620.003309:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d01683800. 00000100:00000010:3.0:1773941620.003313:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d20300000. 00000100:00000010:3.0:1773941620.003316:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d01681800. 00000100:00000010:3.0:1773941620.003319:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d20306000. 00000100:00000010:3.0:1773941620.003322:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d01682000. 00000100:00000010:3.0:1773941620.003326:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d20302000. 00000100:00000010:3.0:1773941620.003330:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d01681400. 00000100:00000010:3.0:1773941620.003333:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d06222000. 00000100:00000010:3.0:1773941620.003358:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d055a4400. 00000100:00000010:3.0:1773941620.003362:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d06226000. 00000100:00000010:3.0:1773941620.003365:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d055a4000. 00000100:00000010:3.0:1773941620.003368:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d06224000. 00000100:00000010:3.0:1773941620.003372:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d055a5400. 00000100:00000010:3.0:1773941620.003374:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d20324000. 00000100:00000010:3.0:1773941620.003378:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1078e400. 00000100:00000010:3.0:1773941620.003381:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d20320000. 00000100:00000010:3.0:1773941620.003384:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1078f400. 00000100:00000010:3.0:1773941620.003387:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d20322000. 00000100:00000010:3.0:1773941620.003390:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1078c000. 00000100:00000010:3.0:1773941620.003398:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d20326000. 00000100:00000010:3.0:1773941620.003401:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1078e800. 00000100:00000010:3.0:1773941620.003404:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d065ba000. 00000100:00000010:3.0:1773941620.003408:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1078cc00. 00000100:00000010:3.0:1773941620.003410:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d065be000. 00000100:00000010:3.0:1773941620.003413:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1078f800. 00000100:00000010:3.0:1773941620.003416:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d065bc000. 00000100:00000010:3.0:1773941620.003419:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1078f000. 00000100:00000010:3.0:1773941620.003422:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d065b8000. 00000100:00000010:3.0:1773941620.003425:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1078dc00. 00000100:00000010:3.0:1773941620.003427:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d05422000. 00000100:00000010:3.0:1773941620.003443:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1078d400. 00000100:00000010:3.0:1773941620.003445:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d05426000. 00000100:00000010:3.0:1773941620.003449:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1078e000. 00000100:00000010:3.0:1773941620.003451:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d05424000. 00000100:00000010:3.0:1773941620.003454:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1078c800. 00000100:00000010:3.0:1773941620.003457:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d05420000. 00000100:00000010:3.0:1773941620.003460:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1078fc00. 00000100:00000010:3.0:1773941620.003462:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08aea000. 00000100:00000010:3.0:1773941620.003466:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1078d000. 00000100:00000010:3.0:1773941620.003469:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08aee000. 00000100:00000010:3.0:1773941620.003472:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1078ec00. 00000100:00000010:3.0:1773941620.003475:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08aec000. 00000100:00000010:3.0:1773941620.003478:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1078c400. 00000100:00000010:3.0:1773941620.003481:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08ae8000. 00000100:00000010:3.0:1773941620.003484:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1078d800. 00000100:00000010:3.0:1773941620.003487:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d18a0e000. 00000100:00000010:3.0:1773941620.003490:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d19727800. 00000100:00000010:3.0:1773941620.003493:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d18a0c000. 00000100:00000010:3.0:1773941620.003497:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d19724400. 00000100:00000010:3.0:1773941620.003499:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d18a08000. 00000100:00000010:3.0:1773941620.003502:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d107d3400. 00000100:00000010:3.0:1773941620.003512:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d18a0a000. 00000100:00000010:3.0:1773941620.003521:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d107d2400. 00000100:00000010:3.0:1773941620.003524:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08a2e000. 00000100:00000010:3.0:1773941620.003546:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d107d0400. 00000100:00000010:3.0:1773941620.003552:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08a2c000. 00000100:00000010:3.0:1773941620.003556:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d107d2c00. 00000100:00000010:3.0:1773941620.003561:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08a28000. 00000100:00000010:3.0:1773941620.003566:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d107d1800. 00000100:00000010:3.0:1773941620.003571:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08a2a000. 00000100:00000010:3.0:1773941620.003576:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d107d1000. 00000100:00000010:3.0:1773941620.003581:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d0713e000. 00000100:00000010:3.0:1773941620.003588:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d107d3c00. 00000100:00000010:3.0:1773941620.003593:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d0713c000. 00000100:00000010:3.0:1773941620.003598:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d107d0800. 00000100:00000010:3.0:1773941620.003604:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d07138000. 00000100:00000010:3.0:1773941620.003609:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d107d2000. 00000100:00000010:3.0:1773941620.003613:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d0713a000. 00000100:00000010:3.0:1773941620.003618:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d107d1400. 00000100:00000010:3.0:1773941620.003623:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d0357c000. 00000100:00000010:3.0:1773941620.003629:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d107d1c00. 00000100:00000010:3.0:1773941620.003633:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d03578000. 00000100:00000010:3.0:1773941620.003638:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d107d3000. 00000100:00000010:3.0:1773941620.003643:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d0357a000. 00000100:00000010:3.0:1773941620.003648:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d107d3800. 00000100:00000010:3.0:1773941620.003653:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d0357e000. 00000100:00000010:3.0:1773941620.003656:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d107d0c00. 00000100:00000010:3.0:1773941620.003659:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d03502000. 00000100:00000010:3.0:1773941620.003684:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d107d2800. 00000100:00000010:3.0:1773941620.003687:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d03506000. 00000100:00000010:3.0:1773941620.003691:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d107d0000. 00000100:00000010:3.0:1773941620.003694:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d03504000. 00000100:00000010:3.0:1773941620.003697:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d203fd800. 00000100:00000010:3.0:1773941620.003700:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d03500000. 00000100:00000010:3.0:1773941620.003703:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d203fd000. 00000100:00000010:3.0:1773941620.003876:0:14694:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff988d20333800. 00000100:00000010:3.0:1773941620.003881:0:14694:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff988d18b1a000. 00000100:00000010:3.0:1773941620.003885:0:14694:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff988d20334800. 00000100:00000001:3.0:1773941620.003888:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003890:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003892:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003894:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003896:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003898:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003899:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003901:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003903:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003904:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003906:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003907:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003908:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003910:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003911:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003913:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003914:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003916:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003917:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003939:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003942:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003943:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003945:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003946:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003948:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003949:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003951:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003952:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003954:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003955:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.003957:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.003958:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004001:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004005:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004014:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004018:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004030:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004032:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004035:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004039:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004041:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004043:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004045:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004046:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004050:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004052:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004056:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004058:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004060:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004062:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004066:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004069:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004072:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004073:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004076:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004078:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004079:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004081:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004083:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004085:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004087:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004089:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004090:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004092:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004093:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004095:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004096:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004098:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004099:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004101:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004102:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004103:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004105:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004106:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004108:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004109:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004116:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004118:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004120:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004121:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004123:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004125:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004127:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004128:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004130:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004131:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004132:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004134:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004135:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004137:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004138:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004140:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004141:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004143:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004144:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004146:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004147:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004149:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004150:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004151:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004153:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004154:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004156:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004157:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004159:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004160:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004162:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004163:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004165:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004166:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004168:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004169:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004170:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004172:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004173:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004175:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004181:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004183:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004184:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004186:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004187:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004189:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004190:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004192:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004193:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004195:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000001:3.0:1773941620.004196:0:14694:0:(service.c:1939:ptlrpc_server_hpreq_fini()) Process entered 00000100:00000001:3.0:1773941620.004197:0:14694:0:(service.c:1950:ptlrpc_server_hpreq_fini()) Process leaving 00000100:00000010:3.0:1773941620.004201:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d10816000. 00000100:00000010:3.0:1773941620.004207:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d203fc400. 00000100:00000010:3.0:1773941620.004210:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d10814000. 00000100:00000010:3.0:1773941620.004213:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d203fec00. 00000100:00000010:3.0:1773941620.004216:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d10810000. 00000100:00000010:3.0:1773941620.004219:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d203ff400. 00000100:00000010:3.0:1773941620.004222:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d10812000. 00000100:00000010:3.0:1773941620.004225:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d203fe400. 00000100:00000010:3.0:1773941620.004228:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d18a30000. 00000100:00000010:3.0:1773941620.004231:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d203fe800. 00000100:00000010:3.0:1773941620.004234:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d18a32000. 00000100:00000010:3.0:1773941620.004237:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d203fc000. 00000100:00000010:3.0:1773941620.004239:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d18a36000. 00000100:00000010:3.0:1773941620.004243:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d203fcc00. 00000100:00000010:3.0:1773941620.004245:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d18a34000. 00000100:00000010:3.0:1773941620.004249:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d203fe000. 00000100:00000010:3.0:1773941620.004252:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d089fa000. 00000100:00000010:3.0:1773941620.004275:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d203fd400. 00000100:00000010:3.0:1773941620.004278:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d089fe000. 00000100:00000010:3.0:1773941620.004281:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d203fdc00. 00000100:00000010:3.0:1773941620.004284:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d089fc000. 00000100:00000010:3.0:1773941620.004287:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d203fc800. 00000100:00000010:3.0:1773941620.004290:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d089f8000. 00000100:00000010:3.0:1773941620.004299:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d203ff800. 00000100:00000010:3.0:1773941620.004302:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08350000. 00000100:00000010:3.0:1773941620.004305:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d203ff000. 00000100:00000010:3.0:1773941620.004308:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08352000. 00000100:00000010:3.0:1773941620.004311:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1127f800. 00000100:00000010:3.0:1773941620.004316:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08356000. 00000100:00000010:3.0:1773941620.004319:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d065b1400. 00000100:00000010:3.0:1773941620.004322:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08354000. 00000100:00000010:3.0:1773941620.004325:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d065b0800. 00000100:00000010:3.0:1773941620.004328:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d089e0000. 00000100:00000010:3.0:1773941620.004331:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d065b2000. 00000100:00000010:3.0:1773941620.004334:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d089e2000. 00000100:00000010:3.0:1773941620.004337:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d065b1000. 00000100:00000010:3.0:1773941620.004340:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d089e6000. 00000100:00000010:3.0:1773941620.004343:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d065b3c00. 00000100:00000010:3.0:1773941620.004346:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d089e4000. 00000100:00000010:3.0:1773941620.004349:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d065b1800. 00000100:00000010:3.0:1773941620.004352:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d03322000. 00000100:00000010:3.0:1773941620.004372:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d065b0400. 00000100:00000010:3.0:1773941620.004375:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d03326000. 00000100:00000010:3.0:1773941620.004378:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d065b2c00. 00000100:00000010:3.0:1773941620.004381:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d03324000. 00000100:00000010:3.0:1773941620.004384:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d065b3400. 00000100:00000010:3.0:1773941620.004387:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d03320000. 00000100:00000010:3.0:1773941620.004391:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d065b2400. 00000100:00000010:3.0:1773941620.004394:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d111fe000. 00000100:00000010:3.0:1773941620.004398:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d065b2800. 00000100:00000010:3.0:1773941620.004400:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d111fc000. 00000100:00000010:3.0:1773941620.004403:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d065b0000. 00000100:00000010:3.0:1773941620.004406:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d111f8000. 00000100:00000010:3.0:1773941620.004409:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d065b0c00. 00000100:00000010:3.0:1773941620.004412:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d111fa000. 00000100:00000010:3.0:1773941620.004415:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d065b3000. 00000100:00000010:3.0:1773941620.004423:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d181ec000. 00000100:00000010:3.0:1773941620.004426:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d065b3800. 00000100:00000010:3.0:1773941620.004428:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d181e8000. 00000100:00000010:3.0:1773941620.004431:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d065b1c00. 00000100:00000010:3.0:1773941620.004434:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d181ea000. 00000100:00000010:3.0:1773941620.004437:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d1a052000. 00000100:00000010:3.0:1773941620.004440:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d181ee000. 00000100:00000010:3.0:1773941620.004443:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d181d0c00. 00000100:00000010:3.0:1773941620.004445:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d05458000. 00000100:00000010:3.0:1773941620.004466:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d181d3000. 00000100:00000010:3.0:1773941620.004470:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d0545a000. 00000100:00000010:3.0:1773941620.004473:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d181d3800. 00000100:00000010:3.0:1773941620.004475:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d0545e000. 00000100:00000010:3.0:1773941620.004479:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d181d0800. 00000100:00000010:3.0:1773941620.004481:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d0545c000. 00000100:00000010:3.0:1773941620.004484:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d181d2000. 00000100:00000010:3.0:1773941620.004487:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08a76000. 00000100:00000010:3.0:1773941620.004491:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d181d1000. 00000100:00000010:3.0:1773941620.004495:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08a74000. 00000100:00000010:3.0:1773941620.004498:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d181d3c00. 00000100:00000010:3.0:1773941620.004501:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08a70000. 00000100:00000010:3.0:1773941620.004504:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d181d2800. 00000100:00000010:3.0:1773941620.004507:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d08a72000. 00000100:00000010:3.0:1773941620.004510:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d181d2400. 00000100:00000010:3.0:1773941620.004512:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d111c6000. 00000100:00000010:3.0:1773941620.004516:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d181d3400. 00000100:00000010:3.0:1773941620.004518:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d111c4000. 00000100:00000010:3.0:1773941620.004521:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d181d2c00. 00000100:00000010:3.0:1773941620.004524:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d111c0000. 00000100:00000010:3.0:1773941620.004529:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d181d0000. 00000100:00000010:3.0:1773941620.004532:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d111c2000. 00000100:00000010:3.0:1773941620.004536:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d181d1400. 00000100:00000010:3.0:1773941620.004538:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d18b90000. 00000100:00000010:3.0:1773941620.004571:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d181d1800. 00000100:00000010:3.0:1773941620.004578:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d18b92000. 00000100:00000010:3.0:1773941620.004583:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d181d0400. 00000100:00000010:3.0:1773941620.004588:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d18b96000. 00000100:00000010:3.0:1773941620.004593:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d11273800. 00000100:00000010:3.0:1773941620.004599:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d18b94000. 00000100:00000010:3.0:1773941620.004604:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d0923dc00. 00000100:00000010:3.0:1773941620.004625:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d203a4000. 00000100:00000010:3.0:1773941620.004632:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d0923c000. 00000100:00000010:3.0:1773941620.004637:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d203a0000. 00000100:00000010:3.0:1773941620.004641:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d0923fc00. 00000100:00000010:3.0:1773941620.004644:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d203a2000. 00000100:00000010:3.0:1773941620.004647:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d0923c400. 00000100:00000010:3.0:1773941620.004650:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d203a6000. 00000100:00000010:3.0:1773941620.004653:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d0923d800. 00000100:00000010:3.0:1773941620.004656:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d18afa000. 00000100:00000010:3.0:1773941620.004660:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d0923c800. 00000100:00000010:3.0:1773941620.004663:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d18afe000. 00000100:00000010:3.0:1773941620.004665:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d0923f800. 00000100:00000010:3.0:1773941620.004668:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d18afc000. 00000100:00000010:3.0:1773941620.004671:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d0923cc00. 00000100:00000010:3.0:1773941620.004674:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d18af8000. 00000100:00000010:3.0:1773941620.004677:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d0923ec00. 00000100:00000010:3.0:1773941620.004680:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d065d2000. 00000100:00000010:3.0:1773941620.004703:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d0923e800. 00000100:00000010:3.0:1773941620.004706:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d065d6000. 00000100:00000010:3.0:1773941620.004709:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d0923e400. 00000100:00000010:3.0:1773941620.004712:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d065d4000. 00000100:00000010:3.0:1773941620.004717:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d0923d000. 00000100:00000010:3.0:1773941620.004720:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d065d0000. 00000100:00000010:3.0:1773941620.004725:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d0923d400. 00000100:00000010:3.0:1773941620.004728:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d2031a000. 00000100:00000010:3.0:1773941620.004732:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d0923f400. 00000100:00000010:3.0:1773941620.004741:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d2031e000. 00000100:00000010:3.0:1773941620.004744:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d03074800. 00000100:00000010:3.0:1773941620.004747:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d2031c000. 00000100:00000010:3.0:1773941620.004750:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d02da7c00. 00000100:00000010:3.0:1773941620.004753:0:14694:0:(service.c:96:ptlrpc_free_rqbd()) kvfree 'rqbd->rqbd_buffer': 8192 at ffff988d20318000. 00000100:00000010:3.0:1773941620.004757:0:14694:0:(service.c:97:ptlrpc_free_rqbd()) kfreed 'rqbd': 968 at ffff988d11258c00. 00000100:00000010:3.0:1773941620.004760:0:14694:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff988d09301000. 00000100:00000010:3.0:1773941620.004764:0:14694:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff988d0544d000. 00000100:00000010:3.0:1773941620.004766:0:14694:0:(service.c:3796:ptlrpc_service_purge_all()) kvfree 'rs': 2048 at ffff988d0544f000. 00000100:00000001:3.0:1773941620.004770:0:14694:0:(nrs.c:1051:nrs_svcpt_cleanup_locked()) Process entered 00000100:00000001:3.0:1773941620.004772:0:14694:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941620.004774:0:14694:0:(nrs.c:111:nrs_policy_stop_locked()) Process entered 00000100:00000001:3.0:1773941620.004776:0:14694:0:(nrs.c:67:nrs_policy_stop0()) Process entered 00000100:00000010:3.0:1773941620.004779:0:14694:0:(nrs_fifo.c:83:nrs_fifo_stop()) kfreed 'head': 40 at ffff988d10872980. 00000100:00000001:3.0:1773941620.004783:0:14694:0:(nrs.c:85:nrs_policy_stop0()) Process leaving 00000100:00000001:3.0:1773941620.004785:0:14694:0:(nrs.c:147:nrs_policy_stop_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941620.004788:0:14694:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941620.004790:0:14694:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d06510c00. 00000100:00000001:3.0:1773941620.004793:0:14694:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941620.004794:0:14694:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941620.004796:0:14694:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d06510540. 00000100:00000001:3.0:1773941620.004798:0:14694:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941620.004800:0:14694:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941620.004801:0:14694:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d06510000. 00000100:00000001:3.0:1773941620.004803:0:14694:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941620.004805:0:14694:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941620.004806:0:14694:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d06510240. 00000100:00000001:3.0:1773941620.004809:0:14694:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941620.004810:0:14694:0:(nrs.c:111:nrs_policy_stop_locked()) Process entered 00000100:00000001:3.0:1773941620.004811:0:14694:0:(nrs.c:67:nrs_policy_stop0()) Process entered 00000100:00000010:3.0:1773941620.004813:0:14694:0:(nrs_fifo.c:83:nrs_fifo_stop()) kfreed 'head': 40 at ffff988d10872e40. 00000100:00000001:3.0:1773941620.004816:0:14694:0:(nrs.c:85:nrs_policy_stop0()) Process leaving 00000100:00000001:3.0:1773941620.004817:0:14694:0:(nrs.c:147:nrs_policy_stop_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941620.004819:0:14694:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941620.004821:0:14694:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d06510900. 00000100:00000001:3.0:1773941620.004823:0:14694:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941620.004824:0:14694:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941620.004826:0:14694:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d06510180. 00000100:00000001:3.0:1773941620.004833:0:14694:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941620.004835:0:14694:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941620.004836:0:14694:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d06510300. 00000100:00000001:3.0:1773941620.004839:0:14694:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941620.004840:0:14694:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941620.004842:0:14694:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d06510480. 00000100:00000010:3.0:1773941620.004844:0:14694:0:(nrs.c:1080:nrs_svcpt_cleanup_locked()) kfreed 'nrs': 112 at ffff988d181fef00. 00000100:00000001:3.0:1773941620.004847:0:14694:0:(nrs.c:1082:nrs_svcpt_cleanup_locked()) Process leaving 00000100:00000001:3.0:1773941620.004848:0:14694:0:(nrs.c:1051:nrs_svcpt_cleanup_locked()) Process entered 00000100:00000001:3.0:1773941620.004850:0:14694:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941620.004851:0:14694:0:(nrs.c:111:nrs_policy_stop_locked()) Process entered 00000100:00000001:3.0:1773941620.004853:0:14694:0:(nrs.c:67:nrs_policy_stop0()) Process entered 00000100:00000010:3.0:1773941620.004855:0:14694:0:(nrs_fifo.c:83:nrs_fifo_stop()) kfreed 'head': 40 at ffff988d10872880. 00000100:00000001:3.0:1773941620.004857:0:14694:0:(nrs.c:85:nrs_policy_stop0()) Process leaving 00000100:00000001:3.0:1773941620.004859:0:14694:0:(nrs.c:147:nrs_policy_stop_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941620.004861:0:14694:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941620.004862:0:14694:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d06510e40. 00000100:00000001:3.0:1773941620.004864:0:14694:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941620.004866:0:14694:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941620.004867:0:14694:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d06510d80. 00000100:00000001:3.0:1773941620.004870:0:14694:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941620.004871:0:14694:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941620.004873:0:14694:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d06510cc0. 00000100:00000001:3.0:1773941620.004875:0:14694:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941620.004876:0:14694:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941620.004878:0:14694:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d065100c0. 00000100:00000001:3.0:1773941620.004880:0:14694:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941620.004881:0:14694:0:(nrs.c:111:nrs_policy_stop_locked()) Process entered 00000100:00000001:3.0:1773941620.004883:0:14694:0:(nrs.c:67:nrs_policy_stop0()) Process entered 00000100:00000010:3.0:1773941620.004884:0:14694:0:(nrs_fifo.c:83:nrs_fifo_stop()) kfreed 'head': 40 at ffff988d10872d00. 00000100:00000001:3.0:1773941620.004887:0:14694:0:(nrs.c:85:nrs_policy_stop0()) Process leaving 00000100:00000001:3.0:1773941620.004889:0:14694:0:(nrs.c:147:nrs_policy_stop_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1773941620.004890:0:14694:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941620.004892:0:14694:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d065106c0. 00000100:00000001:3.0:1773941620.004894:0:14694:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941620.004895:0:14694:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941620.004897:0:14694:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d06510f00. 00000100:00000001:3.0:1773941620.004899:0:14694:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941620.004901:0:14694:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941620.004902:0:14694:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d065103c0. 00000100:00000001:3.0:1773941620.004909:0:14694:0:(nrs.c:714:nrs_policy_unregister()) Process entered 00000100:00000001:3.0:1773941620.004910:0:14694:0:(nrs.c:750:nrs_policy_unregister()) Process leaving 00000100:00000010:3.0:1773941620.004912:0:14694:0:(nrs.c:758:nrs_policy_unregister()) kfreed 'policy': 152 at ffff988d06510a80. 00000100:00000010:3.0:1773941620.004914:0:14694:0:(nrs.c:1080:nrs_svcpt_cleanup_locked()) kfreed 'nrs': 112 at ffff988d181fe080. 00000100:00000001:3.0:1773941620.004916:0:14694:0:(nrs.c:1082:nrs_svcpt_cleanup_locked()) Process leaving 00000020:00000010:0.0:1773941620.005853:0:14694:0:(lprocfs_status.c:1548:stats_free()) kfreed 'stats->ls_cnt_header': 3680 at ffff988d07207000. 00000020:00000010:0.0:1773941620.005874:0:14694:0:(lprocfs_status.c:1558:stats_free()) kfreed 'stats': 608 at ffff988d11259400. 00000100:00000010:0.0:1773941620.005948:0:14694:0:(service.c:3818:ptlrpc_service_free()) kfreed 'array->paa_reqs_array': 12080 at ffff988d2036c000. 00000100:00000010:0.0:1773941620.006007:0:14694:0:(service.c:3824:ptlrpc_service_free()) kfreed 'array->paa_reqs_count': 3020 at ffff988d07202000. 00000100:00000010:0.0:1773941620.006015:0:14694:0:(service.c:3818:ptlrpc_service_free()) kfreed 'array->paa_reqs_array': 12080 at ffff988d203f8000. 00000100:00000010:0.0:1773941620.006027:0:14694:0:(service.c:3824:ptlrpc_service_free()) kfreed 'array->paa_reqs_count': 3020 at ffff988d07200000. 00000100:00000010:0.0:1773941620.006032:0:14694:0:(service.c:3830:ptlrpc_service_free()) kfreed 'svcpt': 784 at ffff988d1081c800. 00000100:00000010:0.0:1773941620.006037:0:14694:0:(service.c:3830:ptlrpc_service_free()) kfreed 'svcpt': 784 at ffff988d203ffc00. 00000100:00000010:0.0:1773941620.006053:0:14694:0:(service.c:3836:ptlrpc_service_free()) kfreed 'svc': 360 at ffff988d063a9c00. 00000100:00000001:0.0:1773941620.006057:0:14694:0:(service.c:3863:ptlrpc_unregister_service()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:3.0:1773941620.006226:0:14707:0:(ldlm_lockd.c:142:have_expired_locks()) Process entered 00010000:00000001:3.0:1773941620.006231:0:14707:0:(ldlm_lockd.c:147:have_expired_locks()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941620.006252:0:14707:0:(lu_object.c:2048:lu_env_remove()) kfreed_rcu 'lei': 40 at ffff988d03ec71c0. 00000020:00000010:3.0:1773941620.006262:0:14707:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d02d18a00. 00000040:00000010:3.0:1773941620.006267:0:14707:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d02d18e00. 00000020:00000010:3.0:1773941620.006270:0:14707:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d0835f000. 00000020:00000010:3.0:1773941620.006275:0:14707:0:(tgt_main.c:739:tgt_key_fini()) slab-freed '(info)': 1288 at ffff988d03345a90. 00000020:00000010:3.0:1773941620.006281:0:14707:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d02d19a00. 80000000:00000010:3.0:1773941620.006286:0:14707:0:(fld_handler.c:35:fld_key_fini()) kfreed 'info': 72 at ffff988d111f7120. 40000000:00000010:3.0:1773941620.006290:0:14707:0:(fid_handler.c:473:seq_key_fini()) kfreed 'info': 48 at ffff988d03ec77c0. 00010000:00000010:0.0:1773941620.006295:0:14694:0:(ldlm_lockd.c:3586:ldlm_cleanup()) kfreed 'ldlm_state': 32 at ffff988d065457a0. 00100000:00000010:3.0:1773941620.006296:0:14707:0:(lfsck_lib.c:43:lfsck_key_fini()) kfreed 'info': 1800 at ffff988d08994800. 00010000:00000001:0.0:1773941620.006324:0:14694:0:(ldlm_lockd.c:3589:ldlm_cleanup()) Process leaving (rc=0 : 0 : 0) 00040000:00000010:3.0:1773941620.006325:0:14707:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d0a21c500. 00010000:00000001:0.0:1773941620.006341:0:14694:0:(ldlm_lockd.c:3058:ldlm_put_ref()) Process leaving 00040000:00000010:3.0:1773941620.006342:0:14707:0:(qmt_dev.c:344:qmt_key_fini()) kfreed 'info': 1256 at ffff988d08992000. 00040000:00000010:3.0:1773941620.006349:0:14707:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d0642d200. 00000004:00000010:3.0:1773941620.006352:0:14707:0:(mdd_device.c:2418:mdd_key_fini()) kfreed 'info': 1872 at ffff988d08990800. 00010000:00000010:0.0:1773941620.006365:0:14694:0:(ldlm_lib.c:647:client_obd_cleanup()) kfreed 'cli->cl_read_io_latency_by_size': 3920 at ffff988d18a20000. 00000004:00000010:3.0:1773941620.006366:0:14707:0:(mdt_handler.c:8422:mdt_key_fini()) kfreed 'info': 2568 at ffff988d02d6b000. 00000004:00000010:3.0:1773941620.006389:0:14707:0:(lod_dev.c:2366:lod_key_fini()) kfreed 'info': 1224 at ffff988d08990000. 00010000:00000010:0.0:1773941620.006389:0:14694:0:(ldlm_lib.c:651:client_obd_cleanup()) kfreed 'cli->cl_write_io_latency_by_size': 3920 at ffff988d18a26000. 00010000:00000001:0.0:1773941620.006421:0:14694:0:(ldlm_lib.c:654:client_obd_cleanup()) Process leaving 00002000:00000010:3.0:1773941620.006422:0:14707:0:(ofd_dev.c:3000:ofd_key_fini()) kfreed 'info': 816 at ffff988d03520400. 10000000:00000010:0.0:1773941620.006429:0:14694:0:(mgc_request.c:834:mgc_device_free()) kfreed 'lu': 64 at ffff988d10872b40. 00000004:00000010:3.0:1773941620.006432:0:14707:0:(osp_dev.c:1819:osp_key_fini()) kfreed 'info': 616 at ffff988d03523400. 10000000:00000001:0.0:1773941620.006437:0:14694:0:(mgc_request.c:836:mgc_device_free()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:3.0:1773941620.006440:0:14707:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d02d18600. 00000020:00000010:0.0:1773941620.006448:0:14694:0:(lu_object.c:566:lu_global_key_fini()) kfreed 'info': 512 at ffff988d03356c00. 00010000:00000001:3.0:1773941620.006449:0:14707:0:(ldlm_lockd.c:290:expired_lock_main()) Process leaving (rc=0 : 0 : 0) 00000040:00000010:0.0:1773941620.006459:0:14694:0:(llog_obd.c:216:llog_key_fini()) kfreed 'info': 480 at ffff988d03357a00. 00000020:00000010:0.0:1773941620.006470:0:14694:0:(dt_object.c:33:dt_global_key_fini()) kfreed 'info': 1264 at ffff988d09305000. 00000020:00000010:0.0:1773941620.006476:0:14694:0:(update_records.c:1191:update_key_fini()) kfreed 'info': 432 at ffff988d03357000. 00040000:00000010:0.0:1773941620.006481:0:14694:0:(lquota_lib.c:35:lquota_key_fini()) kfreed 'info': 248 at ffff988d0a0db600. 00040000:00000010:0.0:1773941620.006488:0:14694:0:(qsd_lib.c:51:qsd_key_fini()) kfreed 'info': 408 at ffff988d03357800. 00000020:00000010:0.0:1773941620.006495:0:14694:0:(lu_object.c:1692:keys_fini()) kfreed 'ctx->lc_value': 320 at ffff988d03356a00. 00000020:00000001:0.0:1773941620.006503:0:14694:0:(obd_class.h:647:obd_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000010:0.0:1773941620.006507:0:14694:0:(genops.c:66:obd_device_free()) slab-freed '(obd)': 6584 at ffff988d2039cd28. 00000020:00000001:0.0:1773941620.006521:0:14694:0:(obd_config.c:923:class_detach()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941620.006525:0:14694:0:(obd_config.c:1655:class_process_config()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000010:0.0:1773941620.006532:0:14694:0:(obd_config.c:2523:class_manual_cleanup()) kfreed 'lcfg': 72 at ffff988d08bbb8a0. 00000020:00000001:0.0:1773941620.006537:0:14694:0:(obd_config.c:2524:class_manual_cleanup()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941620.006541:0:14694:0:(obd_mount.c:620:lustre_stop_mgc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941620.006546:0:14694:0:(obd_mount.c:720:lustre_put_lsi()) Process entered 00000020:01000004:0.0:1773941620.006548:0:14694:0:(obd_mount.c:724:lustre_put_lsi()) put ffff988d111b1800 1 00000020:00000001:0.0:1773941620.006551:0:14694:0:(obd_mount.c:679:lustre_free_lsi()) Process entered 00000020:01000004:0.0:1773941620.006553:0:14694:0:(obd_mount.c:682:lustre_free_lsi()) Freeing lsi ffff988d111b0000 00000020:00000010:0.0:1773941620.006558:0:14694:0:(obd_mount.c:691:lustre_free_lsi()) kfreed 'lsi': 1880 at ffff988d111b0000. 00000020:00000001:0.0:1773941620.006562:0:14694:0:(obd_mount.c:693:lustre_free_lsi()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941620.006564:0:14694:0:(obd_mount.c:727:lustre_put_lsi()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1773941620.006567:0:14694:0:(obd_mount.c:929:lustre_common_put_super()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1773941620.006571:0:14694:0:(cl_object.c:885:cl_env_cache_purge()) Process entered 00000020:00000001:0.0:1773941620.006576:0:14694:0:(cl_object.c:903:cl_env_cache_purge()) Process leaving (rc=4294967295 : 4294967295 : ffffffff) 00000080:00000001:0.0:1773941620.006580:0:14694:0:(llite_lib.c:1639:ll_put_super()) Process leaving 00000080:00000001:0.0:1773941620.006583:0:14694:0:(llite_lib.c:1538:ll_fill_super()) Process leaving (rc=18446744073709551611 : -5 : fffffffffffffffb) 00000080:00020000:0.0:1773941620.006589:0:14694:0:(super25.c:186:lustre_fill_super()) llite: Unable to mount : rc = -5 00000020:00000010:0.0:1773941620.012708:0:14694:0:(obd_mount.c:631:lustre_put_lsm_free()) kfreed 'lmd->lmd_dev': 48 at ffff988d108727c0. 00000020:00000010:0.0:1773941620.012715:0:14694:0:(obd_mount.c:632:lustre_put_lsm_free()) kfreed 'lmd->lmd_profile': 14 at ffff988d065cd330. 00000020:00000010:0.0:1773941620.012723:0:14694:0:(obd_mount.c:640:lustre_put_lsm_free()) kfreed 'lmd->lmd_mgsname': 19 at ffff988d08957120. 00000020:00000010:0.0:1773941620.012745:0:14694:0:(obd_mount.c:642:lustre_put_lsm_free()) kfreed 'lmd->lmd_params': 4096 at ffff988d18a25000. 00000020:00000010:0.0:1773941620.012752:0:14694:0:(obd_mount.c:644:lustre_put_lsm_free()) kfreed 'lmd': 128 at ffff988d181fe800. 00000001:00000001:2.0:1773941620.687396:0:14949:0:(debug.c:644:libcfs_debug_mark_buffer()) ************************************************** 00000001:02000400:2.0:1773941620.687401:0:14949:0:(debug.c:645:libcfs_debug_mark_buffer()) DEBUG MARKER: conf-sanity test_161b: @@@@@@ FAIL: mount with hostname failed 00000001:00000001:2.0:1773941620.689924:0:14949:0:(debug.c:647:libcfs_debug_mark_buffer()) ************************************************** 00000100:00000040:1.0:1773941621.307031:0:14962:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778110 (cpu 0) has reached its deadline 4295011587, at 4295011648 00000100:00000010:1.0:1773941621.307045:0:14962:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d065805a0. 00000100:00000040:0.0:1773941621.308005:0:41:0:(connection.c:75:cpu_latency_work()) work item of ffff988d19778000 (cpu 0) has reached its deadline 4295011587, at 4295011649 00000100:00000010:0.0:1773941621.308015:0:41:0:(connection.c:94:cpu_latency_work()) kfreed 'pm_qos_req_done': 72 at ffff988d06580b40. Debug log: 72368 lines, 72368 kept, 0 dropped, 0 bad.